diff --git a/.github/filters.yaml b/.github/filters.yaml index 528b7342c220..7f7e5a3ec9ce 100644 --- a/.github/filters.yaml +++ b/.github/filters.yaml @@ -357,6 +357,10 @@ planner: - 'components/src/dynamo/profiler/**' - 'components/src/dynamo/aisimulate/**' - 'components/src/dynamo/global_router/**' + # Re-run the SKU contract when either compared operator input changes. + - 'deploy/operator/hack/test_aisimulate_hardware_catalog.py' + - 'deploy/operator/internal/aisimulate/hardware_skus.txt' + - 'deploy/operator/api/v1beta2/dynamographdeploymentrequest_types.go' # Keep this tombstone until the vendored-tree deletion has merged so the # deleted paths in that PR remain covered by the Planner CI filter. - 'aisimulate/**' diff --git a/deploy/helm/charts/platform/components/operator/files/role.yaml b/deploy/helm/charts/platform/components/operator/files/role.yaml index 1ec01f3f9264..17f81925f010 100644 --- a/deploy/helm/charts/platform/components/operator/files/role.yaml +++ b/deploy/helm/charts/platform/components/operator/files/role.yaml @@ -24,9 +24,10 @@ rules: - "" resources: - nodes - - pods/log verbs: - get + - list + - watch - apiGroups: - "" resources: @@ -48,6 +49,12 @@ rules: - list - patch - watch +- apiGroups: + - "" + resources: + - pods/log + verbs: + - get - apiGroups: - "" resources: diff --git a/deploy/operator/api/v1beta2/dynamographdeploymentrequest_types.go b/deploy/operator/api/v1beta2/dynamographdeploymentrequest_types.go index d7413eb3936a..794acf91f039 100644 --- a/deploy/operator/api/v1beta2/dynamographdeploymentrequest_types.go +++ b/deploy/operator/api/v1beta2/dynamographdeploymentrequest_types.go @@ -44,24 +44,44 @@ const ( // GPUHardwareSpec describes accelerator identity and the maximum allocation per candidate. type GPUHardwareSpec struct { // SKU identifies the accelerator using the AISimulate hardware catalog name. - // When omitted, admission discovers the accelerator from the cluster. + // Known values in the bundled AISimulate release are `a100_pcie`, `a100_sxm`, + // `a30`, `b200_sxm`, `b300_sxm`, `b60`, `gb200`, `gb300`, `h100_pcie`, + // `h100_sxm`, `h200_sxm`, `l4`, `l40s`, and `rtx_pro_6000_server`. + // The field remains open so an external AISimulate systems catalog can add identifiers. + // Admission passes an unknown identifier through unchanged and warns when `budget` and the + // parent `numGpusPerNode` are also explicit. Without both, it rejects the identifier because + // cluster discovery cannot associate capacity or topology with it. When omitted from a + // nonempty hardware object, admission discovers the accelerator. // +optional // +kubebuilder:validation:MinLength=1 SKU *string `json:"sku,omitempty"` // Budget is the maximum total number of GPUs one candidate may use. - // When omitted, admission derives the available GPU budget from the cluster. + // When omitted for a known SKU, admission derives the available GPU budget from the cluster. + // An SKU unknown to the bundled AISimulate catalog requires an explicit `budget` and an + // explicit `numGpusPerNode` in the parent hardware object. // +optional // +kubebuilder:validation:Minimum=1 Budget *int32 `json:"budget,omitempty"` } -// HardwareSpec bounds the accelerator configurations evaluated by the search. +// HardwareSpec bounds the accelerator configurations evaluated by the search. An exactly empty +// object opts out of operator cluster discovery and typed hardware defaults. type HardwareSpec struct { // GPU describes accelerator identity and the maximum allocation per candidate. - // Omitted fields are resolved independently by admission. + // In a nonempty hardware object, admission resolves an omitted SKU or budget from the cluster. // +optional GPU *GPUHardwareSpec `json:"gpu,omitempty"` + + // NumGPUsPerNode is the number of GPUs available to one candidate on each node. It lowers + // to AISimulate's num_gpus_per_node input. + // An explicit value is authoritative. In a nonempty hardware object, admission discovers + // an omitted value from the selected cluster node shape. The Run controller never discovers + // this value. An exactly empty hardware object opts out of all operator hardware discovery + // and typed hardware defaults, leaving AISimulate input and defaults authoritative. + // +optional + // +kubebuilder:validation:Minimum=1 + NumGPUsPerNode *int32 `json:"numGpusPerNode,omitempty"` } // KVLoadRatio is either a pinned load ratio or a minimum/maximum range. @@ -237,7 +257,9 @@ type NativeSearchInput struct { Backends []Backend `json:"backends,omitempty"` // Hardware bounds the accelerator configurations evaluated by the search. - // Admission discovers any omitted hardware values from the cluster. + // When omitted, admission discovers SKU, budget and node width. An exactly empty object opts + // out of operator hardware discovery and typed defaults. Admission completes every nonempty + // object with any missing hardware values; explicit values remain authoritative. // +optional Hardware *HardwareSpec `json:"hardware,omitempty"` diff --git a/deploy/operator/api/v1beta2/dynamographdeploymentrun_types.go b/deploy/operator/api/v1beta2/dynamographdeploymentrun_types.go index 692520aab96b..7627dc2730a6 100644 --- a/deploy/operator/api/v1beta2/dynamographdeploymentrun_types.go +++ b/deploy/operator/api/v1beta2/dynamographdeploymentrun_types.go @@ -26,6 +26,7 @@ import ( // DynamoGraphDeploymentRunSpec is an immutable, resolved copy of an accepted DGDR spec. // Run creators resolve optional Request inputs before persisting the Run. // +kubebuilder:validation:XValidation:rule="has(self.model)",message="model is required" +// +kubebuilder:validation:XValidation:rule="has(self.hardware) && ((!has(self.hardware.gpu) && !has(self.hardware.numGpusPerNode)) || (has(self.hardware.gpu) && has(self.hardware.gpu.sku) && has(self.hardware.gpu.budget) && has(self.hardware.numGpusPerNode)))",message="hardware must be exactly empty or fully resolved" type DynamoGraphDeploymentRunSpec struct { NativeSearchInput `json:",inline"` } diff --git a/deploy/operator/api/v1beta2/zz_generated.deepcopy.go b/deploy/operator/api/v1beta2/zz_generated.deepcopy.go index 2188565147ed..9565ea300cd4 100644 --- a/deploy/operator/api/v1beta2/zz_generated.deepcopy.go +++ b/deploy/operator/api/v1beta2/zz_generated.deepcopy.go @@ -542,6 +542,11 @@ func (in *HardwareSpec) DeepCopyInto(out *HardwareSpec) { *out = new(GPUHardwareSpec) (*in).DeepCopyInto(*out) } + if in.NumGPUsPerNode != nil { + in, out := &in.NumGPUsPerNode, &out.NumGPUsPerNode + *out = new(int32) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HardwareSpec. diff --git a/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentrequests.yaml b/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentrequests.yaml index 9f5cb6b112b7..9fa94ef3d825 100644 --- a/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentrequests.yaml +++ b/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentrequests.yaml @@ -9625,27 +9625,49 @@ spec: hardware: description: |- Hardware bounds the accelerator configurations evaluated by the search. - Admission discovers any omitted hardware values from the cluster. + When omitted, admission discovers SKU, budget and node width. An exactly empty object opts + out of operator hardware discovery and typed defaults. Admission completes every nonempty + object with any missing hardware values; explicit values remain authoritative. properties: gpu: description: |- GPU describes accelerator identity and the maximum allocation per candidate. - Omitted fields are resolved independently by admission. + In a nonempty hardware object, admission resolves an omitted SKU or budget from the cluster. properties: budget: description: |- Budget is the maximum total number of GPUs one candidate may use. - When omitted, admission derives the available GPU budget from the cluster. + When omitted for a known SKU, admission derives the available GPU budget from the cluster. + An SKU unknown to the bundled AISimulate catalog requires an explicit `budget` and an + explicit `numGpusPerNode` in the parent hardware object. format: int32 minimum: 1 type: integer sku: description: |- SKU identifies the accelerator using the AISimulate hardware catalog name. - When omitted, admission discovers the accelerator from the cluster. + Known values in the bundled AISimulate release are `a100_pcie`, `a100_sxm`, + `a30`, `b200_sxm`, `b300_sxm`, `b60`, `gb200`, `gb300`, `h100_pcie`, + `h100_sxm`, `h200_sxm`, `l4`, `l40s`, and `rtx_pro_6000_server`. + The field remains open so an external AISimulate systems catalog can add identifiers. + Admission passes an unknown identifier through unchanged and warns when `budget` and the + parent `numGpusPerNode` are also explicit. Without both, it rejects the identifier because + cluster discovery cannot associate capacity or topology with it. When omitted from a + nonempty hardware object, admission discovers the accelerator. minLength: 1 type: string type: object + numGpusPerNode: + description: |- + NumGPUsPerNode is the number of GPUs available to one candidate on each node. It lowers + to AISimulate's num_gpus_per_node input. + An explicit value is authoritative. In a nonempty hardware object, admission discovers + an omitted value from the selected cluster node shape. The Run controller never discovers + this value. An exactly empty hardware object opts out of all operator hardware discovery + and typed hardware defaults, leaving AISimulate input and defaults authoritative. + format: int32 + minimum: 1 + type: integer type: object model: description: Model identifies the model whose deployment configurations are evaluated. diff --git a/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentruns.yaml b/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentruns.yaml index 8218fea7e98f..084ec8a800cb 100644 --- a/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentruns.yaml +++ b/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentruns.yaml @@ -97,27 +97,49 @@ spec: hardware: description: |- Hardware bounds the accelerator configurations evaluated by the search. - Admission discovers any omitted hardware values from the cluster. + When omitted, admission discovers SKU, budget and node width. An exactly empty object opts + out of operator hardware discovery and typed defaults. Admission completes every nonempty + object with any missing hardware values; explicit values remain authoritative. properties: gpu: description: |- GPU describes accelerator identity and the maximum allocation per candidate. - Omitted fields are resolved independently by admission. + In a nonempty hardware object, admission resolves an omitted SKU or budget from the cluster. properties: budget: description: |- Budget is the maximum total number of GPUs one candidate may use. - When omitted, admission derives the available GPU budget from the cluster. + When omitted for a known SKU, admission derives the available GPU budget from the cluster. + An SKU unknown to the bundled AISimulate catalog requires an explicit `budget` and an + explicit `numGpusPerNode` in the parent hardware object. format: int32 minimum: 1 type: integer sku: description: |- SKU identifies the accelerator using the AISimulate hardware catalog name. - When omitted, admission discovers the accelerator from the cluster. + Known values in the bundled AISimulate release are `a100_pcie`, `a100_sxm`, + `a30`, `b200_sxm`, `b300_sxm`, `b60`, `gb200`, `gb300`, `h100_pcie`, + `h100_sxm`, `h200_sxm`, `l4`, `l40s`, and `rtx_pro_6000_server`. + The field remains open so an external AISimulate systems catalog can add identifiers. + Admission passes an unknown identifier through unchanged and warns when `budget` and the + parent `numGpusPerNode` are also explicit. Without both, it rejects the identifier because + cluster discovery cannot associate capacity or topology with it. When omitted from a + nonempty hardware object, admission discovers the accelerator. minLength: 1 type: string type: object + numGpusPerNode: + description: |- + NumGPUsPerNode is the number of GPUs available to one candidate on each node. It lowers + to AISimulate's num_gpus_per_node input. + An explicit value is authoritative. In a nonempty hardware object, admission discovers + an omitted value from the selected cluster node shape. The Run controller never discovers + this value. An exactly empty hardware object opts out of all operator hardware discovery + and typed hardware defaults, leaving AISimulate input and defaults authoritative. + format: int32 + minimum: 1 + type: integer type: object model: description: Model identifies the model whose deployment configurations are evaluated. @@ -281,6 +303,8 @@ spec: x-kubernetes-validations: - message: model is required rule: has(self.model) + - message: hardware must be exactly empty or fully resolved + rule: has(self.hardware) && ((!has(self.hardware.gpu) && !has(self.hardware.numGpusPerNode)) || (has(self.hardware.gpu) && has(self.hardware.gpu.sku) && has(self.hardware.gpu.budget) && has(self.hardware.numGpusPerNode))) status: description: DynamoGraphDeploymentRunStatus represents one search execution's observed state. properties: diff --git a/deploy/operator/hack/test_aisimulate_hardware_catalog.py b/deploy/operator/hack/test_aisimulate_hardware_catalog.py new file mode 100644 index 000000000000..d4bd24083524 --- /dev/null +++ b/deploy/operator/hack/test_aisimulate_hardware_catalog.py @@ -0,0 +1,93 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Keep DGDR's hardware hints synchronized with the pinned AISimulate wheel.""" + +from __future__ import annotations + +import re +import tomllib +from importlib import metadata +from pathlib import Path + +import pytest +import yaml +from packaging.requirements import Requirement +from packaging.utils import canonicalize_name +from packaging.version import Version + +pytestmark = [ + pytest.mark.unit, + pytest.mark.gpu_0, + pytest.mark.pre_merge, + pytest.mark.planner, + pytest.mark.parallel, +] + +REPO_ROOT = Path(__file__).resolve().parents[3] +CATALOG_PATH = REPO_ROOT / "deploy/operator/internal/aisimulate/hardware_skus.txt" +API_PATH = ( + REPO_ROOT / "deploy/operator/api/v1beta2/dynamographdeploymentrequest_types.go" +) +PYPROJECT_PATH = REPO_ROOT / "pyproject.toml" + + +def _pinned_version() -> Version: + with PYPROJECT_PATH.open("rb") as stream: + dependencies = tomllib.load(stream)["project"]["dependencies"] + requirements = [ + Requirement(dependency) + for dependency in dependencies + if canonicalize_name(Requirement(dependency).name) == "aisimulate" + ] + assert len(requirements) == 1, "pyproject.toml must contain one AISimulate pin" + + requirement = requirements[0] + specifiers = list(requirement.specifier) + assert requirement.url is None + assert len(specifiers) == 1 and specifiers[0].operator == "==" + return Version(specifiers[0].version) + + +def _packaged_hardware_skus() -> list[str]: + distribution = metadata.distribution("aisimulate") + system_files = [ + path + for path in distribution.files or [] + if path.parent.as_posix() == "aisimulate_core/systems" + and path.suffix == ".yaml" + ] + result = [] + for packaged_path in system_files: + with distribution.locate_file(packaged_path).open(encoding="utf-8") as stream: + document = yaml.safe_load(stream) + if isinstance(document, dict) and "gpu" in document and "node" in document: + result.append(packaged_path.stem) + return sorted(result) + + +def _documented_hardware_skus() -> list[str]: + source = API_PATH.read_text() + match = re.search( + r"Known values in the bundled AISimulate release are (.*?)\n\s*// The field remains open", + source, + re.DOTALL, + ) + assert match is not None, "GPUHardwareSpec.SKU must document the bundled catalog" + return sorted(re.findall(r"`([^`]+)`", match.group(1))) + + +def _operator_hardware_skus() -> list[str]: + return sorted( + line.strip() + for line in CATALOG_PATH.read_text().splitlines() + if line.strip() and not line.lstrip().startswith("#") + ) + + +def test_dgdr_hardware_catalog_matches_pinned_aisimulate() -> None: + assert Version(metadata.version("aisimulate")) == _pinned_version() + + expected = _packaged_hardware_skus() + assert _operator_hardware_skus() == expected + assert _documented_hardware_skus() == expected diff --git a/deploy/operator/internal/aisimulate/hardware.go b/deploy/operator/internal/aisimulate/hardware.go new file mode 100644 index 000000000000..13c9af441feb --- /dev/null +++ b/deploy/operator/internal/aisimulate/hardware.go @@ -0,0 +1,50 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package aisimulate + +import ( + _ "embed" + "strings" +) + +// hardwareSKUs mirrors the system specifications packaged in the pinned AISimulate wheel. +// It is a compatibility hint for warnings and discovery, not an API enum: explicit +// identifiers outside this set remain valid and are passed through unchanged. +// +//go:embed hardware_skus.txt +var hardwareSKUsText string + +var hardwareSKUs = func() map[string]struct{} { + result := map[string]struct{}{} + for line := range strings.SplitSeq(hardwareSKUsText, "\n") { + value := strings.TrimSpace(line) + if value == "" || strings.HasPrefix(value, "#") { + continue + } + result[value] = struct{}{} + } + return result +}() + +// IsKnownHardwareSKU reports whether sku is packaged by the AISimulate release pinned by +// this Dynamo checkout. False does not make a value invalid: an external system catalog may +// support additional identifiers. +func IsKnownHardwareSKU(sku string) bool { + _, found := hardwareSKUs[sku] + return found +} diff --git a/deploy/operator/internal/aisimulate/hardware_skus.txt b/deploy/operator/internal/aisimulate/hardware_skus.txt new file mode 100644 index 000000000000..e55bc7594a89 --- /dev/null +++ b/deploy/operator/internal/aisimulate/hardware_skus.txt @@ -0,0 +1,17 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +a100_pcie +a100_sxm +a30 +b200_sxm +b300_sxm +b60 +gb200 +gb300 +h100_pcie +h100_sxm +h200_sxm +l4 +l40s +rtx_pro_6000_server diff --git a/deploy/operator/internal/aisimulate/hardware_test.go b/deploy/operator/internal/aisimulate/hardware_test.go new file mode 100644 index 000000000000..3a50b6d33d30 --- /dev/null +++ b/deploy/operator/internal/aisimulate/hardware_test.go @@ -0,0 +1,29 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package aisimulate + +import "testing" + +func TestHardwareCatalogIgnoresComments(t *testing.T) { + if !IsKnownHardwareSKU("h200_sxm") { + t.Fatal("h200_sxm must be present in the bundled hardware catalog") + } + if IsKnownHardwareSKU("#") || IsKnownHardwareSKU("SPDX-License-Identifier:") { + t.Fatal("catalog comments must not be interpreted as hardware SKUs") + } +} diff --git a/deploy/operator/internal/controller/suite_envtest_test.go b/deploy/operator/internal/controller/suite_envtest_test.go index 0a6c4633cc74..585d840d7f2d 100644 --- a/deploy/operator/internal/controller/suite_envtest_test.go +++ b/deploy/operator/internal/controller/suite_envtest_test.go @@ -35,7 +35,7 @@ const controllerAISimulateImage = "registry.example/aisimulate:1.2.0" type fixedHardwareDiscoverer struct{} func (fixedHardwareDiscoverer) Discover(context.Context, nvidiacomv1beta1.GPUSKUType) (*gpu.GPUInfo, error) { - return &gpu.GPUInfo{System: "h200_sxm", GPUsPerNode: 8, NodesWithGPUs: 1}, nil + return &gpu.GPUInfo{System: "h200_sxm", GPUsPerNode: 8, NodesWithGPUs: 1, TotalGPUs: 8, SelectedShapeGPUs: 8}, nil } var ( diff --git a/deploy/operator/internal/gpu/discovery.go b/deploy/operator/internal/gpu/discovery.go index 27f224c958a5..469eb1025efe 100644 --- a/deploy/operator/internal/gpu/discovery.go +++ b/deploy/operator/internal/gpu/discovery.go @@ -164,24 +164,25 @@ var gpuRules = []gpuRule{ // GPUInfo contains discovered GPU configuration from cluster nodes type GPUInfo struct { - NodeName string // Name of the node with this GPU configuration - GPUsPerNode int // Maximum GPUs per node found in the cluster - NodesWithGPUs int // Number of nodes that have GPUs - Model string // GPU product name (e.g., "H100-SXM5-80GB") - VRAMPerGPU int // VRAM in MiB per GPU - System nvidiacomv1beta1.GPUSKUType // AIC hardware system identifier (e.g., "h100_sxm", "h200_sxm"), empty if unknown - MIGEnabled bool // True if MIG is enabled (inferred from model or additional labels, not implemented in this version) - MIGProfiles map[string]int // Optional: map of MIG profile name to count (requires additional label parsing, not implemented in this version) - CloudProvider string // aws | gcp | aks | other | unknown - RDMAEnabled bool // Indicates whether RDMA is enabled for this node (e.g., via InfiniBand, RoCE, or similar high-speed networking) - RDMAType string // Type of RDMA transport detected (e.g., "infiniband", "roce", "rdma", "sriov", or "none") - Interconnect string // Primary GPU-to-GPU interconnect technology used within the node (e.g., "nvlink" for high-bandwidth links or "pcie" for standard bus-based communication) - InterconnectTier string // Qualitative or platform-specific classification of the interconnect (e.g., NVLink generation, topology tier, or vendor-defined performance level) - NVLinkLinks int // Number of NVLink connections per GPU (0 if NVLink is not present or interconnect is PCIe-only) - // OtherModels lists the GPU models of the nodes that unfiltered discovery left out because - // their SKU differs from System. Discovery selects the node with the most GPUs, so a non-empty - // list means System is a choice among several GPU types, not a fact about the cluster. It is - // empty for a filtered discovery and for a cluster of one SKU. + NodeName string // Name of the node with this GPU configuration + GPUsPerNode int // Maximum GPUs per node with the selected SKU + NodesWithGPUs int // Number of discovered nodes with the selected SKU + TotalGPUs int // Sum of GPUs across discovered nodes with the selected SKU, before any budget cap + SelectedShapeGPUs int // Sum of GPUs on nodes matching the selected SKU, GPU count, and VRAM per GPU + Model string // GPU product name (e.g., "H100-SXM5-80GB") + VRAMPerGPU int // VRAM in MiB per GPU + System nvidiacomv1beta1.GPUSKUType // AIC hardware system identifier (e.g., "h100_sxm", "h200_sxm"), empty if unknown + MIGEnabled bool // True if MIG is enabled (inferred from model or additional labels, not implemented in this version) + MIGProfiles map[string]int // Optional: map of MIG profile name to count (requires additional label parsing, not implemented in this version) + CloudProvider string // aws | gcp | aks | other | unknown + RDMAEnabled bool // Indicates whether RDMA is enabled for this node (e.g., via InfiniBand, RoCE, or similar high-speed networking) + RDMAType string // Type of RDMA transport detected (e.g., "infiniband", "roce", "rdma", "sriov", or "none") + Interconnect string // Primary GPU-to-GPU interconnect technology used within the node (e.g., "nvlink" for high-bandwidth links or "pcie" for standard bus-based communication) + InterconnectTier string // Qualitative or platform-specific classification of the interconnect (e.g., NVLink generation, topology tier, or vendor-defined performance level) + NVLinkLinks int // Number of NVLink connections per GPU (0 if NVLink is not present or interconnect is PCIe-only) + // OtherModels lists the GPU models that unfiltered discovery left out because their GPU + // identity differs from the selected type. Known models are grouped by SKU and unknown models + // by normalized model name. It is empty for filtered discovery and a cluster of one GPU type. OtherModels []string } @@ -402,18 +403,21 @@ func (g *GPUDiscovery) discoverGPUsFromDCGMFilteredUncached(ctx context.Context, return nil, fmt.Errorf("no GPU metrics could be parsed from any DCGM pod") } - // Count only nodes with the same SKU as the selected best node, + // Count nodes and GPUs with the same GPU identity as the selected best node, // and detect RDMA on matching nodes only. On a cold cache and a no-RDMA // cluster, this performs one Node read per matching node; that keeps a // single negative node from masking RDMA on another node. nodesWithGPUs := 0 + totalGPUs := 0 + selectedShapeGPUs := 0 var rdmaDetected bool var rdmaType string for _, n := range allNodes { - if n.sku != bestSKU { + if !matchesSelectedGPUType(n.sku, bestSKU, n.info.Model, bestNode.Model) { continue } nodesWithGPUs++ + totalGPUs, selectedShapeGPUs = accumulateGPUCapacity(totalGPUs, selectedShapeGPUs, n.info, bestNode) if !rdmaDetected { rdma, rType := detectRDMAFromNode(ctx, k8sClient, n.nodeName) if rdma { @@ -435,11 +439,13 @@ func (g *GPUDiscovery) discoverGPUsFromDCGMFilteredUncached(ctx context.Context, if err != nil { cloudProvider = CloudProviderUnknown } - bestNode.OtherModels = collectOtherModels(filterSKU, bestSKU, len(allNodes), + bestNode.OtherModels = collectOtherModels(filterSKU, bestSKU, bestNode.Model, len(allNodes), func(i int) (nvidiacomv1beta1.GPUSKUType, string) { return allNodes[i].sku, allNodes[i].info.Model }) bestNode.System = bestSKU bestNode.CloudProvider = cloudProvider bestNode.NodesWithGPUs = nodesWithGPUs + bestNode.TotalGPUs = totalGPUs + bestNode.SelectedShapeGPUs = selectedShapeGPUs bestNode.RDMAEnabled = rdmaDetected bestNode.RDMAType = rdmaType @@ -861,16 +867,19 @@ func DiscoverGPUsFiltered(ctx context.Context, k8sClient client.Reader, filterSK len(nodeList.Items), LabelGPUCount, LabelGPUProduct, LabelGPUMemory) } - // Count only nodes with the same SKU as the selected best node, + // Count nodes and GPUs with the same GPU identity as the selected best node, // and detect RDMA on matching nodes only. On a cold cache and a no-RDMA // cluster, this performs one Node read per matching node; that keeps a // single negative node from masking RDMA on another node. nodesWithGPUs := 0 + totalGPUs := 0 + selectedShapeGPUs := 0 var rdmaDetected bool var rdmaType string for _, n := range allNodes { - if n.sku == bestSKU { + if matchesSelectedGPUType(n.sku, bestSKU, n.info.Model, bestNode.Model) { nodesWithGPUs++ + totalGPUs, selectedShapeGPUs = accumulateGPUCapacity(totalGPUs, selectedShapeGPUs, n.info, bestNode) if !rdmaDetected { rdma, rType := detectRDMAFromNode(ctx, k8sClient, n.info.NodeName) if rdma { @@ -889,16 +898,19 @@ func DiscoverGPUsFiltered(ctx context.Context, k8sClient client.Reader, filterSK rdmaDetected = true } - bestNode.OtherModels = collectOtherModels(filterSKU, bestSKU, len(allNodes), + bestNode.OtherModels = collectOtherModels(filterSKU, bestSKU, bestNode.Model, len(allNodes), func(i int) (nvidiacomv1beta1.GPUSKUType, string) { return allNodes[i].sku, allNodes[i].info.Model }) bestNode.System = bestSKU bestNode.NodesWithGPUs = nodesWithGPUs + bestNode.TotalGPUs = totalGPUs + bestNode.SelectedShapeGPUs = selectedShapeGPUs bestNode.RDMAEnabled = rdmaDetected bestNode.RDMAType = rdmaType logger.Info("GPU discovery completed", "gpusPerNode", bestNode.GPUsPerNode, "nodesWithGPUs", bestNode.NodesWithGPUs, - "totalGpus", bestNode.GPUsPerNode*bestNode.NodesWithGPUs, + "totalGpus", bestNode.TotalGPUs, + "selectedShapeGpus", bestNode.SelectedShapeGPUs, "model", bestNode.Model, "vram", bestNode.VRAMPerGPU, "system", bestNode.System, @@ -907,6 +919,21 @@ func DiscoverGPUsFiltered(ctx context.Context, k8sClient client.Reader, filterSK return bestNode, nil } +func accumulateGPUCapacity(total, selected int, node, selectedShape *GPUInfo) (int, int) { + total += node.GPUsPerNode + if node.GPUsPerNode == selectedShape.GPUsPerNode && node.VRAMPerGPU == selectedShape.VRAMPerGPU { + selected += node.GPUsPerNode + } + return total, selected +} + +func matchesSelectedGPUType(discoveredSKU, selectedSKU nvidiacomv1beta1.GPUSKUType, discoveredModel, selectedModel string) bool { + if discoveredSKU != selectedSKU { + return false + } + return selectedSKU != "" || normalize(discoveredModel) == normalize(selectedModel) +} + // extractGPUInfoFromNode extracts GPU information from a single node's labels. // Returns error if required labels are missing or invalid. func extractGPUInfoFromNode(node *corev1.Node) (*GPUInfo, error) { diff --git a/deploy/operator/internal/gpu/discovery_test.go b/deploy/operator/internal/gpu/discovery_test.go index 2733e20bcd6a..855745afece5 100644 --- a/deploy/operator/internal/gpu/discovery_test.go +++ b/deploy/operator/internal/gpu/discovery_test.go @@ -50,6 +50,53 @@ func newFakeClient(objs ...client.Object) client.Reader { Build() } +func TestGPUDiscoverySelectedShapeCapacity(t *testing.T) { + for _, source := range []string{"node labels", "DCGM"} { + t.Run(source, func(t *testing.T) { + objects := []client.Object{ + &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: "large", Labels: map[string]string{ + LabelGPUCount: "8", LabelGPUProduct: "H100-SXM5-80GB", LabelGPUMemory: "81920", + }}}, + &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: "small", Labels: map[string]string{ + LabelGPUCount: "4", LabelGPUProduct: "H100-SXM5-80GB", LabelGPUMemory: "81920", + }}}, + } + metrics := map[string]GPUInfo{} + for index, node := range []string{"large", "small"} { + ip := fmt.Sprintf("10.0.0.%d", index+1) + count := 8 - 4*index + objects = append(objects, &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{Name: "dcgm-" + node, Namespace: "gpu-operator", Labels: map[string]string{LabelApp: LabelValueNvidiaDCGMExporter}}, + Spec: corev1.PodSpec{NodeName: node}, + Status: corev1.PodStatus{Phase: corev1.PodRunning, PodIP: ip}, + }) + metrics[buildDCGMEndpoint(ip)] = GPUInfo{GPUsPerNode: count, Model: "H100-SXM5-80GB", VRAMPerGPU: 81920} + } + + var info *GPUInfo + var err error + k8sClient := newFakeClient(objects...) + if source == "DCGM" { + discovery := NewGPUDiscovery(func(_ context.Context, endpoint string) (*GPUInfo, error) { + metric, ok := metrics[endpoint] + if !ok { + return nil, fmt.Errorf("unexpected endpoint %q", endpoint) + } + return &metric, nil + }) + info, err = discovery.DiscoverGPUsFromDCGM(t.Context(), k8sClient, nil) + } else { + info, err = DiscoverGPUs(t.Context(), k8sClient) + } + require.NoError(t, err) + assert.Equal(t, 8, info.GPUsPerNode) + assert.Equal(t, 2, info.NodesWithGPUs) + assert.Equal(t, 12, info.TotalGPUs) + assert.Equal(t, 8, info.SelectedShapeGPUs) + }) + } +} + func TestDiscoverGPUs_SingleNode(t *testing.T) { ctx := context.Background() diff --git a/deploy/operator/internal/gpu/heterogeneous.go b/deploy/operator/internal/gpu/heterogeneous.go index 312f9e362695..4dd3bcdcab05 100644 --- a/deploy/operator/internal/gpu/heterogeneous.go +++ b/deploy/operator/internal/gpu/heterogeneous.go @@ -22,28 +22,38 @@ import ( nvidiacomv1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" ) -// collectOtherModels returns, sorted and without repeats, the models of the nodes whose SKU -// differs from the selected one. A filtered discovery asked for one SKU and returns nothing: -// the nodes of other SKUs were never candidates. at yields the SKU and the model of node i. +// collectOtherModels returns, sorted and without repeats, the models of nodes whose GPU identity +// differs from the selected one. It uses the same identity rule as capacity aggregation: known +// GPUs are grouped by SKU and unknown GPUs by normalized model name. A filtered discovery asked +// for one SKU and returns nothing because nodes of other SKUs were never candidates. func collectOtherModels( - filterSKU, selected nvidiacomv1beta1.GPUSKUType, + filterSKU, selectedSKU nvidiacomv1beta1.GPUSKUType, + selectedModel string, nodes int, at func(i int) (nvidiacomv1beta1.GPUSKUType, string), ) []string { if filterSKU != "" { return nil } - seen := map[string]struct{}{} + seen := map[string]string{} for i := 0; i < nodes; i++ { - if sku, model := at(i); sku != selected { - seen[model] = struct{}{} + sku, model := at(i) + if matchesSelectedGPUType(sku, selectedSKU, model, selectedModel) { + continue + } + identity := "sku:" + string(sku) + if sku == "" { + identity = "model:" + normalize(model) + } + if existing, exists := seen[identity]; !exists || model < existing { + seen[identity] = model } } if len(seen) == 0 { return nil } models := make([]string, 0, len(seen)) - for model := range seen { + for _, model := range seen { models = append(models, model) } sort.Strings(models) diff --git a/deploy/operator/internal/gpu/heterogeneous_test.go b/deploy/operator/internal/gpu/heterogeneous_test.go index 71380d227077..c07dfc1451b9 100644 --- a/deploy/operator/internal/gpu/heterogeneous_test.go +++ b/deploy/operator/internal/gpu/heterogeneous_test.go @@ -29,24 +29,33 @@ func TestCollectOtherModels(t *testing.T) { model string } tests := []struct { - name string - filter nvidiacomv1beta1.GPUSKUType - selected nvidiacomv1beta1.GPUSKUType - nodes []node - want []string + name string + filter nvidiacomv1beta1.GPUSKUType + selectedSKU nvidiacomv1beta1.GPUSKUType + selectedModel string + nodes []node + want []string }{ - {"a cluster of one SKU", "", "h200_sxm", []node{{"h200_sxm", "H200"}, {"h200_sxm", "H200"}}, nil}, - {"a second SKU is reported once, sorted", "", "h200_sxm", + {"a cluster of one SKU", "", "h200_sxm", "H200", []node{{"h200_sxm", "H200"}, {"h200_sxm", "H200"}}, nil}, + {"a second SKU is reported once, sorted", "", "h200_sxm", "H200", []node{{"h200_sxm", "H200"}, {"h100_sxm", "H100"}, {"b200_sxm", "B200"}, {"h100_sxm", "H100"}}, []string{"B200", "H100"}}, - {"a model the catalog does not know still counts", "", "h200_sxm", + {"a model the catalog does not know still counts", "", "h200_sxm", "H200", []node{{"h200_sxm", "H200"}, {"", "Mystery GPU"}}, []string{"Mystery GPU"}}, - {"a filtered discovery never reports", "h200_sxm", "h200_sxm", + {"distinct models unknown to the catalog remain distinct", "", "", "Mystery Alpha", + []node{{"", "Mystery Alpha"}, {"", "Mystery Beta"}}, []string{"Mystery Beta"}}, + {"normalized spellings of an unknown selected model share its identity", "", "", "Mystery GPU", + []node{{"", "Mystery GPU"}, {"", "mystery-gpu"}}, nil}, + {"normalized spellings of another unknown model are reported once", "", "", "Mystery Alpha", + []node{{"", "Mystery Alpha"}, {"", "mystery-beta"}, {"", "Mystery Beta"}}, []string{"Mystery Beta"}}, + {"known models with the same SKU are reported once", "", "h200_sxm", "H200", + []node{{"h200_sxm", "H200"}, {"h100_sxm", "NVIDIA H100"}, {"h100_sxm", "H100 SXM"}}, []string{"H100 SXM"}}, + {"a filtered discovery never reports", "h200_sxm", "h200_sxm", "H200", []node{{"h200_sxm", "H200"}, {"h100_sxm", "H100"}}, nil}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got := collectOtherModels(tt.filter, tt.selected, len(tt.nodes), + got := collectOtherModels(tt.filter, tt.selectedSKU, tt.selectedModel, len(tt.nodes), func(i int) (nvidiacomv1beta1.GPUSKUType, string) { return tt.nodes[i].sku, tt.nodes[i].model }) if !reflect.DeepEqual(got, tt.want) { t.Fatalf("got %v, want %v", got, tt.want) diff --git a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2.go b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2.go index 37e7098880b3..1398aa10a878 100644 --- a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2.go +++ b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2.go @@ -50,9 +50,10 @@ const ( // API version because the canonical names belong to v1beta1. // // Admission owns every v1beta2 default, and each default has this one owner: backends, -// objective.mode, the scalar metric, recommendation.maxCandidates, workload.static, the -// cluster hardware and the AISimulate image. workload.static: {} opts out of the static -// workload. objective.sla has no default: an omitted SLA stays absent. +// objective.mode, the scalar metric, recommendation.maxCandidates, workload.static, discovered +// cluster hardware and the AISimulate image. workload.static: {} opts out of the static workload. +// objective.sla has no default: an omitted SLA stays absent. hardware: {} opts out of cluster +// discovery and typed hardware defaults; a nonempty object is completed. // spec.aisimulate.parameters never changes a default. It overrides the defaulted typed // values later, when the input is lowered. @@ -121,7 +122,7 @@ func decodeOldSpecV1beta2( // dynamoGraphDeploymentRequestV1beta2DefaultOptions carries the request-scoped values the // traversal needs. An empty field means the environment had nothing to offer. type dynamoGraphDeploymentRequestV1beta2DefaultOptions struct { - // hardware is the GPU identity and budget discovered from the cluster. + // hardware is the GPU identity, budget and node width discovered from the cluster. hardware hardwareDefaultsV1beta2 // aisimulateImage is the operator's resolved AISimulate image. aisimulateImage string @@ -131,8 +132,9 @@ type dynamoGraphDeploymentRequestV1beta2DefaultOptions struct { // hardwareDefaultsV1beta2 is the hardware the cluster can offer for the values a Request omits. type hardwareDefaultsV1beta2 struct { - sku string - budget int32 + sku string + budget int32 + numGPUsPerNode int32 } // defaultDynamoGraphDeploymentRequestSpecV1beta2 defaults spec in place. It applies on @@ -162,7 +164,8 @@ func defaultNativeSearchInputV1beta2( input.Backends = []nvidiacomv1beta2.Backend{"vllm", "sglang", "trtllm"} } - // Hardware has no static default: it exists only when the cluster supplied some. + // Omitted hardware receives discovered values. An explicit empty object reaches this + // traversal with empty hardware options and remains the opt-out object. if input.Hardware == nil && opts.hardware != (hardwareDefaultsV1beta2{}) { input.Hardware = &nvidiacomv1beta2.HardwareSpec{} } @@ -202,6 +205,9 @@ func defaultHardwareSpecV1beta2(hardware *nvidiacomv1beta2.HardwareSpec, default if hardware.GPU != nil { defaultGPUHardwareSpecV1beta2(hardware.GPU, defaults) } + if hardware.NumGPUsPerNode == nil && defaults.numGPUsPerNode > 0 { + hardware.NumGPUsPerNode = ptr.To(defaults.numGPUsPerNode) + } } // defaultGPUHardwareSpecV1beta2 fills each omitted value on its own, so a caller can pin the diff --git a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_hardware_test.go b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_hardware_test.go index d4cbbc621047..b9698b2b5792 100644 --- a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_hardware_test.go +++ b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_hardware_test.go @@ -20,6 +20,8 @@ package defaulting import ( "context" "errors" + "math" + "strconv" "strings" "testing" @@ -28,58 +30,84 @@ import ( admissionv1 "k8s.io/api/admission/v1" ) -func hardwareOfV1beta2(sku *string, budget *int32) *nvidiacomv1beta2.HardwareSpec { - return &nvidiacomv1beta2.HardwareSpec{GPU: &nvidiacomv1beta2.GPUHardwareSpec{SKU: sku, Budget: budget}} +func hardwareOfV1beta2(sku *string, budget *int32, numGPUsPerNode ...*int32) *nvidiacomv1beta2.HardwareSpec { + hardware := &nvidiacomv1beta2.HardwareSpec{GPU: &nvidiacomv1beta2.GPUHardwareSpec{SKU: sku, Budget: budget}} + if len(numGPUsPerNode) > 0 { + hardware.NumGPUsPerNode = numGPUsPerNode[0] + } + return hardware } func TestResolveHardwareCompletesTheTypedHardwareOrRejects(t *testing.T) { str := func(s string) *string { return &s } num := func(n int32) *int32 { return &n } - h200x8 := gpu.GPUInfo{System: "h200_sxm", GPUsPerNode: 8, NodesWithGPUs: 1} - h200x64 := gpu.GPUInfo{System: "h200_sxm", GPUsPerNode: 8, NodesWithGPUs: 8} - tests := []struct { + h200x8 := gpu.GPUInfo{System: "h200_sxm", GPUsPerNode: 8, NodesWithGPUs: 1, TotalGPUs: 8, SelectedShapeGPUs: 8} + h200x64 := gpu.GPUInfo{System: "h200_sxm", GPUsPerNode: 8, NodesWithGPUs: 8, TotalGPUs: 64, SelectedShapeGPUs: 64} + type testCase struct { name string hardware *nvidiacomv1beta2.HardwareSpec found gpu.GPUInfo err error wantSKU string wantBudget int32 + wantWidth int32 wantCalls int wantFilter string wantErr string - }{ - {"omitted hardware discovers the SKU and the budget", nil, h200x8, nil, "h200_sxm", 8, 1, "", ""}, - {"an empty hardware object is automatic discovery", &nvidiacomv1beta2.HardwareSpec{}, h200x8, nil, "h200_sxm", 8, 1, "", ""}, + } + tests := []testCase{ + {"omitted hardware discovers the SKU budget and node width", nil, h200x8, nil, "h200_sxm", 8, 8, 1, "", ""}, + {"an empty hardware object opts out of discovery", &nvidiacomv1beta2.HardwareSpec{}, gpu.GPUInfo{}, errors.New("discovery must not run"), "", 0, 0, 0, "", ""}, + {"same-SKU nodes with different GPU counts use the selected node shape", nil, + gpu.GPUInfo{System: "h200_sxm", GPUsPerNode: 8, NodesWithGPUs: 2, TotalGPUs: 12, SelectedShapeGPUs: 8}, + nil, "h200_sxm", 8, 8, 1, "", ""}, {"an empty gpu object is automatic discovery", - &nvidiacomv1beta2.HardwareSpec{GPU: &nvidiacomv1beta2.GPUHardwareSpec{}}, h200x8, nil, "h200_sxm", 8, 1, "", ""}, - {"the discovered budget is capped", nil, h200x64, nil, "h200_sxm", 32, 1, "", ""}, + &nvidiacomv1beta2.HardwareSpec{GPU: &nvidiacomv1beta2.GPUHardwareSpec{}}, h200x8, nil, "h200_sxm", 8, 8, 1, "", ""}, + {"the discovered budget is capped", nil, h200x64, nil, "h200_sxm", 32, 8, 1, "", ""}, {"a SKU alone keeps the SKU, filters the cluster and persists the capped budget", - hardwareOfV1beta2(str("h200_sxm"), nil), h200x64, nil, "", 32, 1, "h200_sxm", ""}, - {"a budget alone keeps the budget and discovers the SKU", hardwareOfV1beta2(nil, num(4)), h200x8, nil, "h200_sxm", 0, 1, "", ""}, - {"an explicit budget above the cluster size is kept", hardwareOfV1beta2(nil, num(64)), h200x8, nil, "h200_sxm", 0, 1, "", ""}, - {"an explicit budget above the cap is kept", hardwareOfV1beta2(str("h200_sxm"), num(64)), h200x8, nil, "", 0, 0, "", ""}, - {"a SKU and a budget never reach the cluster", hardwareOfV1beta2(str("h100_sxm"), num(4)), gpu.GPUInfo{}, errors.New("down"), "", 0, 0, "", ""}, - {"a failed discovery rejects", nil, gpu.GPUInfo{}, errors.New("no GPUs found"), "", 0, 1, "", + hardwareOfV1beta2(str("h200_sxm"), nil), h200x64, nil, "", 32, 8, 1, "h200_sxm", ""}, + {"a budget alone keeps the budget and discovers the SKU", hardwareOfV1beta2(nil, num(4)), h200x8, nil, "h200_sxm", 0, 8, 1, "", ""}, + {"an explicit budget above the cluster size is kept", hardwareOfV1beta2(nil, num(64)), h200x8, nil, "h200_sxm", 0, 8, 1, "", ""}, + {"an explicit SKU and budget discover the missing node width", hardwareOfV1beta2(str("h200_sxm"), num(64)), h200x8, nil, "", 0, 8, 1, "h200_sxm", ""}, + {"fully explicit hardware bypasses discovery", hardwareOfV1beta2(str("h100_sxm"), num(4), num(8)), gpu.GPUInfo{}, errors.New("down"), "", 0, 0, 0, "", ""}, + {"fully explicit unknown hardware is authoritative", hardwareOfV1beta2(str("future_accelerator"), num(4), num(2)), gpu.GPUInfo{}, errors.New("down"), "", 0, 0, 0, "", ""}, + {"an unknown SKU and budget require an explicit node width", hardwareOfV1beta2(str("future_accelerator"), num(4)), h200x8, nil, "", 0, 0, 0, "", + `spec.hardware.numGpusPerNode: Required value: cannot discover a node width for SKU "future_accelerator" because it is not known to the bundled AISimulate catalog; set spec.hardware.numGpusPerNode explicitly`}, + {"an unknown SKU alone requires explicit capacity and topology", hardwareOfV1beta2(str("future_accelerator"), nil), h200x8, nil, "", 0, 0, 0, "", + `spec.hardware.gpu.budget: Required value: cannot discover GPU capacity or topology for SKU "future_accelerator" because it is not known to the bundled AISimulate catalog; set spec.hardware.gpu.budget and spec.hardware.numGpusPerNode explicitly`}, + {"a failed discovery rejects", nil, gpu.GPUInfo{}, errors.New("no GPUs found"), "", 0, 0, 1, "", "spec.hardware.gpu: Required value: the cluster's GPUs could not be discovered (no GPUs found)"}, {"a GPU model the catalog does not know rejects", nil, - gpu.GPUInfo{Model: "Mystery GPU", GPUsPerNode: 8, NodesWithGPUs: 1}, nil, "", 0, 1, "", + gpu.GPUInfo{Model: "Mystery GPU", GPUsPerNode: 8, NodesWithGPUs: 1}, nil, "", 0, 0, 1, "", `spec.hardware.gpu.sku: Required value: the cluster's GPU model "Mystery GPU" maps to no AISimulate SKU`}, + {"a legacy mapping outside the bundled catalog rejects", nil, + gpu.GPUInfo{System: "gb10", Model: "GB10", GPUsPerNode: 1, NodesWithGPUs: 1, TotalGPUs: 1, SelectedShapeGPUs: 1}, nil, "", 0, 0, 1, "", + `spec.hardware.gpu.sku: Required value: the cluster's GPU model "GB10" maps to no AISimulate SKU`}, {"no GPUs of the SKU reject the missing budget", hardwareOfV1beta2(str("h100_sxm"), nil), - gpu.GPUInfo{System: "h100_sxm"}, nil, "", 0, 1, "h100_sxm", + gpu.GPUInfo{System: "h100_sxm"}, nil, "", 0, 0, 1, "h100_sxm", `spec.hardware.gpu.budget: Required value: no GPUs of SKU "h100_sxm" were found in the cluster`}, {"a heterogeneous cluster rejects an omitted SKU", nil, gpu.GPUInfo{System: "h200_sxm", Model: "H200", GPUsPerNode: 8, NodesWithGPUs: 1, OtherModels: []string{"H100"}}, - nil, "", 0, 1, "", + nil, "", 0, 0, 1, "", "spec.hardware.gpu.sku: Required value: the cluster has several GPU types (H200, H100); set spec.hardware.gpu.sku"}, {"a heterogeneous cluster rejects a budget alone too", hardwareOfV1beta2(nil, num(4)), gpu.GPUInfo{System: "h200_sxm", Model: "H200", GPUsPerNode: 8, NodesWithGPUs: 1, OtherModels: []string{"H100"}}, - nil, "", 0, 1, "", + nil, "", 0, 0, 1, "", "spec.hardware.gpu.sku: Required value: the cluster has several GPU types (H200, H100)"}, {"a pinned SKU is not ambiguous in a heterogeneous cluster", hardwareOfV1beta2(str("h200_sxm"), nil), - gpu.GPUInfo{System: "h200_sxm", GPUsPerNode: 8, NodesWithGPUs: 1}, nil, "", 8, 1, "h200_sxm", ""}, - {"no GPUs at all reject the missing budget", nil, gpu.GPUInfo{System: "h200_sxm"}, nil, "", 0, 1, "", + gpu.GPUInfo{System: "h200_sxm", GPUsPerNode: 8, NodesWithGPUs: 1, TotalGPUs: 8, SelectedShapeGPUs: 8}, nil, "", 8, 8, 1, "h200_sxm", ""}, + {"no GPUs at all reject the missing budget", nil, gpu.GPUInfo{System: "h200_sxm"}, nil, "", 0, 0, 1, "", `spec.hardware.gpu.budget: Required value: no GPUs of SKU "h200_sxm" were found in the cluster`}, } + if strconv.IntSize > 32 { + tooManyGPUs := int64(math.MaxInt32) + 1 + tests = append(tests, testCase{ + name: "a discovered node width must fit the API integer", + found: gpu.GPUInfo{System: "h200_sxm", GPUsPerNode: int(tooManyGPUs), SelectedShapeGPUs: 8}, + wantCalls: 1, + wantErr: "spec.hardware.numGpusPerNode: Invalid value: 2147483648: discovered node width exceeds the API maximum of 2147483647", + }) + } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { probe := &fakeDiscoverer{found: tt.found, err: tt.err} @@ -93,8 +121,8 @@ func TestResolveHardwareCompletesTheTypedHardwareOrRejects(t *testing.T) { } else if err != nil { t.Fatalf("unexpected error: %v", err) } - if err == nil && (got.sku != tt.wantSKU || got.budget != tt.wantBudget) { - t.Fatalf("defaults = %+v, want sku %q budget %d", got, tt.wantSKU, tt.wantBudget) + if err == nil && (got.sku != tt.wantSKU || got.budget != tt.wantBudget || got.numGPUsPerNode != tt.wantWidth) { + t.Fatalf("defaults = %+v, want sku %q budget %d numGpusPerNode %d", got, tt.wantSKU, tt.wantBudget, tt.wantWidth) } if probe.calls != tt.wantCalls { t.Fatalf("discovery calls = %d, want %d", probe.calls, tt.wantCalls) @@ -110,8 +138,40 @@ func TestResolveHardwareLeavesAnUnchangedUpdateAlone(t *testing.T) { probe := &fakeDiscoverer{err: errors.New("down")} spec := &nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec{} got, err := DGDRV1beta2Options{Hardware: probe}.resolveHardware(context.Background(), admissionv1.Update, spec.DeepCopy(), spec) - if err != nil || got.sku != "" || got.budget != 0 || probe.calls != 0 { + if err != nil || got.sku != "" || got.budget != 0 || got.numGPUsPerNode != 0 || probe.calls != 0 { t.Fatalf("got %+v, %v after %d calls; an update that leaves spec.hardware unchanged must not read the cluster", got, err, probe.calls) } } + +func TestDefaultHardwareSpecV1beta2(t *testing.T) { + t.Run("preserves the explicit empty-object opt-out", func(t *testing.T) { + hardware := &nvidiacomv1beta2.HardwareSpec{} + defaultHardwareSpecV1beta2(hardware, hardwareDefaultsV1beta2{}) + if hardware.GPU != nil || hardware.NumGPUsPerNode != nil { + t.Fatalf("hardware = %#v, want the empty-object opt-out unchanged", hardware) + } + }) + + t.Run("persists every discovered field", func(t *testing.T) { + hardware := &nvidiacomv1beta2.HardwareSpec{} + defaultHardwareSpecV1beta2(hardware, hardwareDefaultsV1beta2{ + sku: "h200_sxm", budget: 8, numGPUsPerNode: 8, + }) + if hardware.GPU == nil || hardware.GPU.SKU == nil || *hardware.GPU.SKU != "h200_sxm" || + hardware.GPU.Budget == nil || *hardware.GPU.Budget != 8 || + hardware.NumGPUsPerNode == nil || *hardware.NumGPUsPerNode != 8 { + t.Fatalf("hardware = %#v, want every discovered field", hardware) + } + }) + + t.Run("preserves an explicit node width", func(t *testing.T) { + hardware := &nvidiacomv1beta2.HardwareSpec{NumGPUsPerNode: new(int32(4))} + defaultHardwareSpecV1beta2(hardware, hardwareDefaultsV1beta2{ + sku: "h200_sxm", budget: 8, numGPUsPerNode: 8, + }) + if hardware.NumGPUsPerNode == nil || *hardware.NumGPUsPerNode != 4 { + t.Fatalf("numGpusPerNode = %v, want explicit value 4", hardware.NumGPUsPerNode) + } + }) +} diff --git a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_helpers.go b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_helpers.go index fe7bfb6c36f9..1ef87169d472 100644 --- a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_helpers.go +++ b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_helpers.go @@ -21,6 +21,7 @@ import ( "context" "errors" "fmt" + "math" "strings" "time" @@ -29,6 +30,7 @@ import ( nvidiacomv1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" nvidiacomv1beta2 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta2" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/aisimulate" "github.com/ai-dynamo/dynamo/deploy/operator/internal/gpu" admissionv1 "k8s.io/api/admission/v1" apiequality "k8s.io/apimachinery/pkg/api/equality" @@ -72,10 +74,12 @@ type DGDRV1beta2Options struct { MaxHardwareBudget int32 } +// +kubebuilder:rbac:groups="",resources=nodes,verbs=get;list;watch + // resolveDefaults reads the environment for the values a v1beta2 spec omits. It does not -// mutate spec. It fails when the typed hardware cannot be completed: the error is the -// admission verdict, so a Request is never stored with hardware that depends on AISimulate's -// implicit defaults or on a cluster that could not be read. +// mutate spec. It fails when a nonempty typed hardware object cannot be completed: the error +// is the admission verdict, so a Request is stored with either exactly empty hardware or all +// three typed hardware values. func (o DGDRV1beta2Options) resolveDefaults( ctx context.Context, operation admissionv1.Operation, @@ -92,8 +96,13 @@ func (o DGDRV1beta2Options) resolveDefaults( }, nil } -// resolveHardware returns the SKU and budget the typed spec omits. It asks the cluster only -// when something is missing, restricted to the typed SKU when there is one. +// resolveHardware returns the SKU, budget and node width that cluster discovery can supply. +// An exactly empty hardware object opts out of all operator discovery and typed hardware +// defaults. Omitted hardware discovers all three values. Every nonempty partial object discovers +// only its missing values; an explicit node width is always preserved by structural defaulting. +// When a known SKU is present, discovery is restricted to it. A SKU outside the bundled +// AISimulate catalog must supply both budget and node width because discovery cannot associate +// cluster capacity or topology with it. // // Discovery runs on CREATE and on an UPDATE that changes spec.hardware. A Request stays // mutable and every Run snapshots it, so a caller may change or remove typed hardware before @@ -103,32 +112,91 @@ func (o DGDRV1beta2Options) resolveDefaults( // the writer's back. oldSpec is the stored spec on UPDATE and nil otherwise. The decision reads // the typed hardware alone, never spec.aisimulate.parameters. // -// After a successful return the typed hardware is complete. An omitted SKU becomes the -// cluster's AISimulate SKU, and an omitted budget becomes min(matching GPUs, cap). An explicit -// budget is authoritative: it is never capped, whatever the cluster holds. What the cluster -// cannot supply is rejected on the field that fixes it: a failed discovery, a GPU model the -// AISimulate catalog does not know, and a SKU without any matching GPU. +// After discovery, an omitted SKU becomes the cluster's AISimulate SKU, an omitted budget +// becomes min(matching GPUs, cap), and an omitted node width becomes the selected node shape. +// Explicit values are authoritative: they are neither rewritten nor capped. What the cluster +// cannot supply is rejected on the omitted SKU, budget or node width that fixes it. func (o DGDRV1beta2Options) resolveHardware( ctx context.Context, operation admissionv1.Operation, oldSpec, spec *nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec, ) (hardwareDefaultsV1beta2, error) { + // Presence distinguishes the explicit empty-object opt-out from omitted hardware. + if isEmptyHardwareSpecV1beta2(spec.Hardware) { + return hardwareDefaultsV1beta2{}, nil + } + changed := operation == admissionv1.Update && oldSpec != nil && !apiequality.Semantic.DeepEqual(oldSpec.Hardware, spec.Hardware) if o.Hardware == nil || (operation != admissionv1.Create && !changed) { return hardwareDefaultsV1beta2{}, nil } - var sku string - var hasBudget bool - if spec.Hardware != nil && spec.Hardware.GPU != nil { - if spec.Hardware.GPU.SKU != nil { - sku = *spec.Hardware.GPU.SKU - } - hasBudget = spec.Hardware.GPU.Budget != nil + intent := hardwareIntentFromSpecV1beta2(spec.Hardware) + if intent.complete() { + return hardwareDefaultsV1beta2{}, nil } - if sku != "" && hasBudget { + // Preserve an explicitly invalid empty SKU for structural validation instead of replacing it. + if intent.hasSKU && intent.sku == "" { return hardwareDefaultsV1beta2{}, nil } + if err := validateDiscoverableHardwareV1beta2(intent); err != nil { + return hardwareDefaultsV1beta2{}, err + } + + found, err := o.discoverHardwareV1beta2(ctx, intent.sku) + if err != nil { + return hardwareDefaultsV1beta2{}, hardwareDiscoveryErrorV1beta2(intent, err) + } + return o.hardwareDefaultsFromDiscoveryV1beta2(intent, found) +} + +type hardwareIntentV1beta2 struct { + sku string + hasSKU bool + hasBudget bool + hasNumGPUsPerNode bool +} + +func hardwareIntentFromSpecV1beta2(hardware *nvidiacomv1beta2.HardwareSpec) hardwareIntentV1beta2 { + var intent hardwareIntentV1beta2 + if hardware == nil { + return intent + } + intent.hasNumGPUsPerNode = hardware.NumGPUsPerNode != nil + if hardware.GPU == nil { + return intent + } + intent.hasBudget = hardware.GPU.Budget != nil + if hardware.GPU.SKU != nil { + intent.hasSKU = true + intent.sku = *hardware.GPU.SKU + } + return intent +} + +func (i hardwareIntentV1beta2) complete() bool { + return i.hasSKU && i.hasBudget && i.hasNumGPUsPerNode +} + +func validateDiscoverableHardwareV1beta2(intent hardwareIntentV1beta2) error { + if intent.sku == "" || aisimulate.IsKnownHardwareSKU(intent.sku) { + return nil + } + hardwarePath := field.NewPath("spec", "hardware") + if !intent.hasBudget { + detail := fmt.Sprintf("cannot discover a GPU budget for SKU %q because it is not known to the bundled AISimulate catalog; set spec.hardware.gpu.budget explicitly", intent.sku) + if !intent.hasNumGPUsPerNode { + detail = fmt.Sprintf("cannot discover GPU capacity or topology for SKU %q because it is not known to the bundled AISimulate catalog; set spec.hardware.gpu.budget and spec.hardware.numGpusPerNode explicitly", intent.sku) + } + return rejectHardwareV1beta2(field.Required(hardwarePath.Child("gpu", "budget"), detail)) + } + return rejectHardwareV1beta2(field.Required( + hardwarePath.Child("numGpusPerNode"), + fmt.Sprintf("cannot discover a node width for SKU %q because it is not known to the bundled AISimulate catalog; set spec.hardware.numGpusPerNode explicitly", intent.sku), + )) +} + +func (o DGDRV1beta2Options) discoverHardwareV1beta2(ctx context.Context, sku string) (*gpu.GPUInfo, error) { timeout := o.HardwareDiscoveryTimeout if timeout <= 0 { @@ -136,17 +204,52 @@ func (o DGDRV1beta2Options) resolveHardware( } ctx, cancel := context.WithTimeout(ctx, timeout) defer cancel() + return o.Hardware.Discover(ctx, nvidiacomv1beta1.GPUSKUType(sku)) +} - gpuPath := field.NewPath("spec", "hardware", "gpu") - found, err := o.Hardware.Discover(ctx, nvidiacomv1beta1.GPUSKUType(sku)) - if err != nil { - return hardwareDefaultsV1beta2{}, rejectHardwareV1beta2(field.Required(gpuPath, fmt.Sprintf( - "the cluster's GPUs could not be discovered (%v); set spec.hardware.gpu.sku and spec.hardware.gpu.budget", err))) +func hardwareDiscoveryErrorV1beta2(intent hardwareIntentV1beta2, err error) error { + hardwarePath := field.NewPath("spec", "hardware") + gpuPath := hardwarePath.Child("gpu") + if intent.sku != "" { + missingPath := gpuPath.Child("budget") + missingField := "spec.hardware.gpu.budget" + if !intent.hasBudget && !intent.hasNumGPUsPerNode { + missingField = "spec.hardware.gpu.budget and spec.hardware.numGpusPerNode" + } + if intent.hasBudget { + missingPath = hardwarePath.Child("numGpusPerNode") + missingField = "spec.hardware.numGpusPerNode" + } + return rejectHardwareV1beta2(field.Required( + missingPath, + fmt.Sprintf("could not be discovered for SKU %q (%v); set %s", intent.sku, err, missingField), + )) } + return rejectHardwareV1beta2(field.Required(gpuPath, fmt.Sprintf( + "the cluster's GPUs could not be discovered (%v); set spec.hardware.gpu.sku, spec.hardware.gpu.budget, and spec.hardware.numGpusPerNode", err))) +} + +func (o DGDRV1beta2Options) hardwareDefaultsFromDiscoveryV1beta2( + intent hardwareIntentV1beta2, + found *gpu.GPUInfo, +) (hardwareDefaultsV1beta2, error) { + hardwarePath := field.NewPath("spec", "hardware") + gpuPath := hardwarePath.Child("gpu") + sku := intent.sku // The cluster facts are mapped into the v1beta2 API here: the AISimulate catalog name of - // the GPU model (empty when the catalog does not know it) and the total GPU count. + // the GPU model (empty when the catalog does not know it) and the selected node-shape capacity. var defaults hardwareDefaultsV1beta2 + if !intent.hasNumGPUsPerNode && found != nil && found.GPUsPerNode > 0 { + if found.GPUsPerNode > math.MaxInt32 { + return hardwareDefaultsV1beta2{}, rejectHardwareV1beta2(field.Invalid( + hardwarePath.Child("numGpusPerNode"), + found.GPUsPerNode, + fmt.Sprintf("discovered node width exceeds the API maximum of %d", math.MaxInt32), + )) + } + defaults.numGPUsPerNode = int32(found.GPUsPerNode) + } if sku == "" { if found != nil && len(found.OtherModels) > 0 { return hardwareDefaultsV1beta2{}, rejectHardwareV1beta2(field.Required(gpuPath.Child("sku"), fmt.Sprintf( @@ -154,7 +257,7 @@ func (o DGDRV1beta2Options) resolveHardware( strings.Join(append([]string{found.Model}, found.OtherModels...), ", ")))) } sku = string(gpu.KnownSKU(found)) - if sku == "" { + if sku == "" || !aisimulate.IsKnownHardwareSKU(sku) { detail := "the cluster's GPUs map to no AISimulate SKU; set spec.hardware.gpu.sku" if found != nil && found.Model != "" { detail = fmt.Sprintf("the cluster's GPU model %q maps to no AISimulate SKU; set spec.hardware.gpu.sku", found.Model) @@ -163,11 +266,10 @@ func (o DGDRV1beta2Options) resolveHardware( } defaults.sku = sku } - if !hasBudget { + if !intent.hasBudget { var gpus int if found != nil { - // TODO(#17192): Use the discovered aggregate GPU count so mixed node sizes of one SKU are counted correctly. - gpus = found.GPUsPerNode * found.NodesWithGPUs + gpus = found.SelectedShapeGPUs } if gpus <= 0 { return hardwareDefaultsV1beta2{}, rejectHardwareV1beta2(field.Required(gpuPath.Child("budget"), fmt.Sprintf( @@ -179,9 +281,19 @@ func (o DGDRV1beta2Options) resolveHardware( } defaults.budget = int32(min(gpus, int(maxBudget))) } + if !intent.hasNumGPUsPerNode && defaults.numGPUsPerNode == 0 { + return hardwareDefaultsV1beta2{}, rejectHardwareV1beta2(field.Required( + hardwarePath.Child("numGpusPerNode"), + fmt.Sprintf("could not be discovered for SKU %q; set spec.hardware.numGpusPerNode", sku), + )) + } return defaults, nil } +func isEmptyHardwareSpecV1beta2(hardware *nvidiacomv1beta2.HardwareSpec) bool { + return hardware != nil && hardware.GPU == nil && hardware.NumGPUsPerNode == nil +} + // hardwareRejectionV1beta2 is hardware the cluster cannot complete, with the field that fixes it. type hardwareRejectionV1beta2 struct{ err *field.Error } diff --git a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_test.go b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_test.go index 25b6dfbf90b5..258ef52120e0 100644 --- a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_test.go +++ b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_test.go @@ -51,13 +51,7 @@ func (f *fakeDiscoverer) Discover(ctx context.Context, filterSKU nvidiacomv1beta } func TestResolveHardwareV1beta2(t *testing.T) { - spec := func(sku *string, budget *int32) *nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec { - s := modelOnlySpecV1beta2() - if sku != nil || budget != nil { - s.Hardware = &nvidiacomv1beta2.HardwareSpec{GPU: &nvidiacomv1beta2.GPUHardwareSpec{SKU: sku, Budget: budget}} - } - return s - } + spec := hardwareSpecV1beta2 t.Run("no discoverer", func(t *testing.T) { if got, err := (DGDRV1beta2Options{}).resolveHardware(context.Background(), admissionv1.Create, nil, spec(nil, nil)); err != nil || !(got == hardwareDefaultsV1beta2{}) { @@ -66,7 +60,7 @@ func TestResolveHardwareV1beta2(t *testing.T) { }) t.Run("complete hardware never asks the cluster", func(t *testing.T) { d := &fakeDiscoverer{found: gpu.GPUInfo{System: "h100_sxm", GPUsPerNode: 4, NodesWithGPUs: 1}} - got, err := DGDRV1beta2Options{Hardware: d}.resolveHardware(context.Background(), admissionv1.Create, nil, spec(v1beta2Ptr("b200_sxm"), v1beta2Ptr(int32(2)))) + got, err := DGDRV1beta2Options{Hardware: d}.resolveHardware(context.Background(), admissionv1.Create, nil, spec(v1beta2Ptr("b200_sxm"), v1beta2Ptr(int32(2)), v1beta2Ptr(int32(4)))) if err != nil { t.Fatal(err) } @@ -75,12 +69,12 @@ func TestResolveHardwareV1beta2(t *testing.T) { } }) t.Run("everything missing", func(t *testing.T) { - d := &fakeDiscoverer{found: gpu.GPUInfo{System: "h200_sxm", GPUsPerNode: 16, NodesWithGPUs: 1}} + d := &fakeDiscoverer{found: gpu.GPUInfo{System: "h200_sxm", GPUsPerNode: 16, NodesWithGPUs: 1, TotalGPUs: 16, SelectedShapeGPUs: 16}} got, err := DGDRV1beta2Options{Hardware: d}.resolveHardware(context.Background(), admissionv1.Create, nil, spec(nil, nil)) if err != nil { t.Fatal(err) } - if got.sku != "h200_sxm" || got.budget != 16 || d.filter != "" { + if got.sku != "h200_sxm" || got.budget != 16 || got.numGPUsPerNode != 16 || d.filter != "" { t.Fatalf("got %+v, filter %q", got, d.filter) } if d.timeout <= 0 || d.timeout > defaultHardwareDiscoveryTimeout { @@ -88,12 +82,12 @@ func TestResolveHardwareV1beta2(t *testing.T) { } }) t.Run("a pinned SKU filters discovery and keeps its value", func(t *testing.T) { - d := &fakeDiscoverer{found: gpu.GPUInfo{System: "b200_sxm", GPUsPerNode: 4, NodesWithGPUs: 1}} + d := &fakeDiscoverer{found: gpu.GPUInfo{System: "b200_sxm", GPUsPerNode: 4, NodesWithGPUs: 1, TotalGPUs: 4, SelectedShapeGPUs: 4}} got, err := DGDRV1beta2Options{Hardware: d}.resolveHardware(context.Background(), admissionv1.Create, nil, spec(v1beta2Ptr("b200_sxm"), nil)) if err != nil { t.Fatal(err) } - if d.filter != "b200_sxm" || got.sku != "" || got.budget != 4 { + if d.filter != "b200_sxm" || got.sku != "" || got.budget != 4 || got.numGPUsPerNode != 4 { t.Fatalf("got %+v, filter %q", got, d.filter) } }) @@ -103,12 +97,12 @@ func TestResolveHardwareV1beta2(t *testing.T) { if err != nil { t.Fatal(err) } - if got.sku != "h200_sxm" || got.budget != 0 { + if got.sku != "h200_sxm" || got.budget != 0 || got.numGPUsPerNode != 64 { t.Fatalf("got %+v", got) } }) t.Run("the budget is capped", func(t *testing.T) { - d := &fakeDiscoverer{found: gpu.GPUInfo{System: "h200_sxm", GPUsPerNode: 1024, NodesWithGPUs: 1}} + d := &fakeDiscoverer{found: gpu.GPUInfo{System: "h200_sxm", GPUsPerNode: 1024, NodesWithGPUs: 1, TotalGPUs: 1024, SelectedShapeGPUs: 1024}} if got, err := (DGDRV1beta2Options{Hardware: d}).resolveHardware(context.Background(), admissionv1.Create, nil, spec(nil, nil)); err != nil || got.budget != 32 { t.Fatalf("default cap: %+v", got) } @@ -126,13 +120,13 @@ func TestResolveHardwareV1beta2(t *testing.T) { } func TestResolveDefaultsV1beta2(t *testing.T) { - d := &fakeDiscoverer{found: gpu.GPUInfo{System: "h200_sxm", GPUsPerNode: 8, NodesWithGPUs: 1}} + d := &fakeDiscoverer{found: gpu.GPUInfo{System: "h200_sxm", GPUsPerNode: 8, NodesWithGPUs: 1, TotalGPUs: 8, SelectedShapeGPUs: 8}} got, err := DGDRV1beta2Options{Hardware: d, AISimulateImage: "registry.example.com/aisimulate:1.2.0"}. resolveDefaults(context.Background(), admissionv1.Create, nil, modelOnlySpecV1beta2()) if err != nil { t.Fatal(err) } - if got.aisimulateImage != "registry.example.com/aisimulate:1.2.0" || got.hardware.sku != "h200_sxm" || got.hardware.budget != 8 { + if got.aisimulateImage != "registry.example.com/aisimulate:1.2.0" || got.hardware.sku != "h200_sxm" || got.hardware.budget != 8 || got.hardware.numGPUsPerNode != 8 { t.Fatalf("got %+v", got) } } @@ -151,7 +145,7 @@ func v1beta2Envelope(t *testing.T, spec *nvidiacomv1beta2.DynamoGraphDeploymentR func TestDefaultEnvelopeV1beta2(t *testing.T) { options := DGDRV1beta2Options{ AISimulateImage: "registry.example.com/aisimulate:1.2.0", - Hardware: &fakeDiscoverer{found: gpu.GPUInfo{System: "h200_sxm", GPUsPerNode: 8, NodesWithGPUs: 1}}, + Hardware: &fakeDiscoverer{found: gpu.GPUInfo{System: "h200_sxm", GPUsPerNode: 8, NodesWithGPUs: 1, TotalGPUs: 8, SelectedShapeGPUs: 8}}, } dgdr := v1beta2Envelope(t, modelOnlySpecV1beta2()) @@ -162,7 +156,7 @@ func TestDefaultEnvelopeV1beta2(t *testing.T) { if err := json.Unmarshal(dgdr.Spec.V1Beta2.Raw, &got); err != nil { t.Fatal(err) } - if got.Hardware == nil || got.Hardware.GPU == nil || *got.Hardware.GPU.SKU != "h200_sxm" || *got.Hardware.GPU.Budget != 8 { + if got.Hardware == nil || got.Hardware.GPU == nil || *got.Hardware.GPU.SKU != "h200_sxm" || *got.Hardware.GPU.Budget != 8 || got.Hardware.NumGPUsPerNode == nil || *got.Hardware.NumGPUsPerNode != 8 { t.Fatalf("hardware not defaulted: %+v", got.Hardware) } if got.AISimulate == nil || got.AISimulate.Image != "registry.example.com/aisimulate:1.2.0" { @@ -190,10 +184,13 @@ func TestDefaultEnvelopeV1beta2RejectsMalformedSpec(t *testing.T) { } } -func hardwareSpecV1beta2(sku *string, budget *int32) *nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec { +func hardwareSpecV1beta2(sku *string, budget *int32, numGPUsPerNode ...*int32) *nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec { spec := modelOnlySpecV1beta2() - if sku != nil || budget != nil { + if sku != nil || budget != nil || len(numGPUsPerNode) > 0 { spec.Hardware = &nvidiacomv1beta2.HardwareSpec{GPU: &nvidiacomv1beta2.GPUHardwareSpec{SKU: sku, Budget: budget}} + if len(numGPUsPerNode) > 0 { + spec.Hardware.NumGPUsPerNode = numGPUsPerNode[0] + } } return spec } @@ -216,7 +213,7 @@ func TestHardwareIsNeverDiscoveredOnOtherOperations(t *testing.T) { func TestResolveHardwareByOperationV1beta2(t *testing.T) { full := func() *nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec { - return hardwareSpecV1beta2(v1beta2Ptr("h100_sxm"), v1beta2Ptr(int32(4))) + return hardwareSpecV1beta2(v1beta2Ptr("h100_sxm"), v1beta2Ptr(int32(4)), v1beta2Ptr(int32(8))) } tests := []struct { name string @@ -227,7 +224,7 @@ func TestResolveHardwareByOperationV1beta2(t *testing.T) { wantFilter string }{ {"create discovers everything", admissionv1.Create, nil, hardwareSpecV1beta2(nil, nil), - hardwareDefaultsV1beta2{sku: "h200_sxm", budget: 8}, 1, ""}, + hardwareDefaultsV1beta2{sku: "h200_sxm", budget: 8, numGPUsPerNode: 8}, 1, ""}, {"metadata-only update without hardware does not discover", admissionv1.Update, hardwareSpecV1beta2(nil, nil), hardwareSpecV1beta2(nil, nil), hardwareDefaultsV1beta2{}, 0, ""}, {"update with unchanged incomplete hardware does not discover", admissionv1.Update, @@ -236,21 +233,21 @@ func TestResolveHardwareByOperationV1beta2(t *testing.T) { {"update without a stored spec does not discover", admissionv1.Update, nil, hardwareSpecV1beta2(nil, nil), hardwareDefaultsV1beta2{}, 0, ""}, {"removing only the budget discovers only the budget", admissionv1.Update, - full(), hardwareSpecV1beta2(v1beta2Ptr("h100_sxm"), nil), + full(), hardwareSpecV1beta2(v1beta2Ptr("h100_sxm"), nil, v1beta2Ptr(int32(8))), hardwareDefaultsV1beta2{budget: 8}, 1, "h100_sxm"}, {"removing only the SKU discovers only the SKU", admissionv1.Update, - full(), hardwareSpecV1beta2(nil, v1beta2Ptr(int32(4))), + full(), hardwareSpecV1beta2(nil, v1beta2Ptr(int32(4)), v1beta2Ptr(int32(8))), hardwareDefaultsV1beta2{sku: "h200_sxm"}, 1, ""}, {"removing all typed hardware discovers both", admissionv1.Update, full(), hardwareSpecV1beta2(nil, nil), - hardwareDefaultsV1beta2{sku: "h200_sxm", budget: 8}, 1, ""}, + hardwareDefaultsV1beta2{sku: "h200_sxm", budget: 8, numGPUsPerNode: 8}, 1, ""}, {"changing to complete hardware does not discover", admissionv1.Update, - full(), hardwareSpecV1beta2(v1beta2Ptr("b200_sxm"), v1beta2Ptr(int32(2))), + full(), hardwareSpecV1beta2(v1beta2Ptr("b200_sxm"), v1beta2Ptr(int32(2)), v1beta2Ptr(int32(4))), hardwareDefaultsV1beta2{}, 0, ""}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - d := &fakeDiscoverer{found: gpu.GPUInfo{System: "h200_sxm", GPUsPerNode: 8, NodesWithGPUs: 1}} + d := &fakeDiscoverer{found: gpu.GPUInfo{System: "h200_sxm", GPUsPerNode: 8, NodesWithGPUs: 1, TotalGPUs: 8, SelectedShapeGPUs: 8}} got, err := DGDRV1beta2Options{Hardware: d}.resolveHardware(context.Background(), tt.op, tt.old, tt.spec) if err != nil { t.Fatal(err) diff --git a/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest.go b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest.go index 85e6b28a1695..080e8405b10e 100644 --- a/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest.go +++ b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest.go @@ -59,6 +59,10 @@ func (v *dynamoGraphDeploymentRequestValidation) warn(message string) { v.warnings = append(v.warnings, message) } +func (v *dynamoGraphDeploymentRequestValidation) warnf(format string, args ...any) { + v.warn(fmt.Sprintf(format, args...)) +} + // Validate performs stateless validation on request. ctx must carry the admission request and // request must not be nil. func (v *DynamoGraphDeploymentRequestValidator) Validate( @@ -123,7 +127,7 @@ func (v *dynamoGraphDeploymentRequestValidation) validateDynamoGraphDeploymentRe // A v1beta2 Request reaches this webhook as a v1beta1 object carrying spec.v1beta2; // the v1beta1 rules below do not apply to it. if spec.V1Beta2 != nil { - return validateDynamoGraphDeploymentRequestEnvelopeV1beta2(spec.V1Beta2, fldPath.Child("v1beta2")) + return v.validateDynamoGraphDeploymentRequestEnvelopeV1beta2(spec.V1Beta2, fldPath.Child("v1beta2")) } allErrs := field.ErrorList{} @@ -290,7 +294,7 @@ func (v *dynamoGraphDeploymentRequestValidation) validateDynamoGraphDeploymentRe // spec runs on every CREATE and UPDATE: an unchanged spec is no reason to skip it. Ratcheting // belongs to the generated AISimulate schema validation, which will ratchet its own result. if newSpec.V1Beta2 != nil { - return validateDynamoGraphDeploymentRequestEnvelopeV1beta2(newSpec.V1Beta2, fldPath.Child("v1beta2")) + return v.validateDynamoGraphDeploymentRequestEnvelopeV1beta2(newSpec.V1Beta2, fldPath.Child("v1beta2")) } allErrs := field.ErrorList{} diff --git a/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_v1beta2.go b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_v1beta2.go index 557129b380e4..982dd2fe153f 100644 --- a/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_v1beta2.go +++ b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_v1beta2.go @@ -21,6 +21,7 @@ import ( "encoding/json" nvidiacomv1beta2 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta2" + aisimulatecatalog "github.com/ai-dynamo/dynamo/deploy/operator/internal/aisimulate" "github.com/ai-dynamo/dynamo/deploy/operator/internal/controller/aisimulate" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" "k8s.io/apimachinery/pkg/util/validation/field" @@ -31,7 +32,7 @@ import ( // does not decode is reported there. Errors about the decoded spec use the v1beta2 spec's own // field paths (spec.model, spec.aisimulate, ...), which are what the user wrote, so they are // rooted at spec rather than at fldPath. -func validateDynamoGraphDeploymentRequestEnvelopeV1beta2( +func (v *dynamoGraphDeploymentRequestValidation) validateDynamoGraphDeploymentRequestEnvelopeV1beta2( raw *apiextensionsv1.JSON, fldPath *field.Path, ) field.ErrorList { @@ -39,6 +40,13 @@ func validateDynamoGraphDeploymentRequestEnvelopeV1beta2( if err := json.Unmarshal(raw.Raw, &spec); err != nil { return field.ErrorList{field.Invalid(fldPath, "", "must be a valid v1beta2 spec: "+err.Error())} } + if spec.Hardware != nil && spec.Hardware.GPU != nil && spec.Hardware.GPU.SKU != nil && + !aisimulatecatalog.IsKnownHardwareSKU(*spec.Hardware.GPU.SKU) { + v.warnf( + "spec.hardware.gpu.sku: %q is not known to the AISimulate release bundled with this operator; the value is passed through unchanged and may work with an external AISimulate systems catalog", + *spec.Hardware.GPU.SKU, + ) + } return validateDynamoGraphDeploymentRequestV1beta2(&spec) } diff --git a/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_v1beta2_test.go b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_v1beta2_test.go index 9ad45a24a0b9..c2dde731821c 100644 --- a/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_v1beta2_test.go +++ b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_v1beta2_test.go @@ -146,17 +146,18 @@ func v1beta2Envelope(raw string) *apiextensionsv1.JSON { func TestValidateEnvelopeV1beta2(t *testing.T) { path := field.NewPath("spec", "v1beta2") + validation := &dynamoGraphDeploymentRequestValidation{} good, err := json.Marshal(v1beta2DefaultedSpec()) if err != nil { t.Fatal(err) } - v1beta2RequireNoError(t, validateDynamoGraphDeploymentRequestEnvelopeV1beta2(v1beta2Envelope(string(good)), path)) + v1beta2RequireNoError(t, validation.validateDynamoGraphDeploymentRequestEnvelopeV1beta2(v1beta2Envelope(string(good)), path)) bad := v1beta2DefaultedSpec() bad.AISimulate = nil raw, _ := json.Marshal(bad) - v1beta2RequireError(t, validateDynamoGraphDeploymentRequestEnvelopeV1beta2(v1beta2Envelope(string(raw)), path), "aisimulate") + v1beta2RequireError(t, validation.validateDynamoGraphDeploymentRequestEnvelopeV1beta2(v1beta2Envelope(string(raw)), path), "aisimulate") - v1beta2RequireError(t, validateDynamoGraphDeploymentRequestEnvelopeV1beta2(v1beta2Envelope(`{"model": 3}`), path), "spec.v1beta2: must be a valid v1beta2 spec") + v1beta2RequireError(t, validation.validateDynamoGraphDeploymentRequestEnvelopeV1beta2(v1beta2Envelope(`{"model": 3}`), path), "spec.v1beta2: must be a valid v1beta2 spec") } diff --git a/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_validation_envtest_test.go b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_validation_envtest_test.go index f9f0f206108d..85b9e0a96e2f 100644 --- a/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_validation_envtest_test.go +++ b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_validation_envtest_test.go @@ -60,7 +60,7 @@ func TestDynamoGraphDeploymentRequestValidator_Validate(t *testing.T) { "spec":{"components":[{"name":"VllmPrefillWorker"}]} }` - discoveredH200 := gpu.GPUInfo{System: "h200_sxm", GPUsPerNode: 8, NodesWithGPUs: 1} + discoveredH200 := gpu.GPUInfo{System: "h200_sxm", GPUsPerNode: 8, NodesWithGPUs: 1, TotalGPUs: 8, SelectedShapeGPUs: 8} tests := []struct { name string @@ -119,6 +119,17 @@ func TestDynamoGraphDeploymentRequestValidator_Validate(t *testing.T) { name: "v1beta2 create bypasses legacy v1beta1 validation", request: beta2DGDRForAdmission(setBeta2AdmissionHardware), }, + { + name: "v1beta2 unknown explicit SKU is preserved with a warning", + request: beta2DGDRForAdmission(func(request *nvidiacomv1beta2.DynamoGraphDeploymentRequest) { + sku, budget, width := "future_accelerator", int32(4), int32(2) + request.Spec.Hardware = &nvidiacomv1beta2.HardwareSpec{ + GPU: &nvidiacomv1beta2.GPUHardwareSpec{SKU: &sku, Budget: &budget}, + NumGPUsPerNode: &width, + } + }), + wantWarnings: []string{`spec.hardware.gpu.sku: "future_accelerator" is not known to the AISimulate release bundled with this operator; the value is passed through unchanged and may work with an external AISimulate systems catalog`}, + }, { name: "v1beta2 update bypasses legacy v1beta1 validation", oldRequest: beta2DGDRForAdmission(setBeta2AdmissionHardware), @@ -561,17 +572,12 @@ func TestDynamoGraphDeploymentRequestValidator_Validate(t *testing.T) { { name: "v1beta2 explicit hardware is kept", request: v1beta2DGDRForAdmission(func(spec map[string]any) { - spec["hardware"] = map[string]any{"gpu": map[string]any{"sku": "h100_sxm", "budget": int64(4)}} + spec["hardware"] = map[string]any{"gpu": map[string]any{"sku": "h100_sxm", "budget": int64(4)}, "numGpusPerNode": int64(4)} }, nil), gpuDiscovery: true, discovered: discoveredH200, check: func(t *testing.T, result *unstructured.Unstructured) { - if got := v1beta2String(t, result, "spec", "hardware", "gpu", "sku"); got != "h100_sxm" { - t.Fatalf("hardware.gpu.sku = %q, want h100_sxm", got) - } - if got := v1beta2Number(t, result, "spec", "hardware", "gpu", "budget"); got != 4 { - t.Fatalf("hardware.gpu.budget = %v, want 4", got) - } + expectV1beta2GPUValues(t, result, "h100_sxm", 4, 4) }, }, { @@ -624,10 +630,10 @@ func TestDynamoGraphDeploymentRequestValidator_Validate(t *testing.T) { { name: "v1beta2 removing only the budget rediscovers only the budget", oldRequest: v1beta2DGDRForAdmission(func(spec map[string]any) { - spec["hardware"] = map[string]any{"gpu": map[string]any{"sku": "h100_sxm", "budget": int64(4)}} + spec["hardware"] = map[string]any{"gpu": map[string]any{"sku": "h100_sxm", "budget": int64(4)}, "numGpusPerNode": int64(8)} }, nil), request: v1beta2DGDRForAdmission(func(spec map[string]any) { - spec["hardware"] = map[string]any{"gpu": map[string]any{"sku": "h100_sxm"}} + spec["hardware"] = map[string]any{"gpu": map[string]any{"sku": "h100_sxm"}, "numGpusPerNode": int64(8)} }, nil), gpuDiscovery: true, discovered: discoveredH200, @@ -638,10 +644,10 @@ func TestDynamoGraphDeploymentRequestValidator_Validate(t *testing.T) { { name: "v1beta2 removing only the SKU rediscovers only the SKU", oldRequest: v1beta2DGDRForAdmission(func(spec map[string]any) { - spec["hardware"] = map[string]any{"gpu": map[string]any{"sku": "h100_sxm", "budget": int64(4)}} + spec["hardware"] = map[string]any{"gpu": map[string]any{"sku": "h100_sxm", "budget": int64(4)}, "numGpusPerNode": int64(8)} }, nil), request: v1beta2DGDRForAdmission(func(spec map[string]any) { - spec["hardware"] = map[string]any{"gpu": map[string]any{"budget": int64(4)}} + spec["hardware"] = map[string]any{"gpu": map[string]any{"budget": int64(4)}, "numGpusPerNode": int64(8)} }, nil), gpuDiscovery: true, discovered: discoveredH200, @@ -652,7 +658,7 @@ func TestDynamoGraphDeploymentRequestValidator_Validate(t *testing.T) { { name: "v1beta2 removing all typed hardware rediscovers the SKU and the budget", oldRequest: v1beta2DGDRForAdmission(func(spec map[string]any) { - spec["hardware"] = map[string]any{"gpu": map[string]any{"sku": "h100_sxm", "budget": int64(4)}} + spec["hardware"] = map[string]any{"gpu": map[string]any{"sku": "h100_sxm", "budget": int64(4)}, "numGpusPerNode": int64(8)} }, nil), request: v1beta2DGDRForAdmission(nil, nil), gpuDiscovery: true, @@ -793,33 +799,55 @@ func TestDynamoGraphDeploymentRequestValidator_Validate(t *testing.T) { name: "v1beta2 hardware omitted is discovered and the budget capped", request: v1beta2DGDRForAdmission(nil, nil), gpuDiscovery: true, - discovered: gpu.GPUInfo{System: "h200_sxm", GPUsPerNode: 8, NodesWithGPUs: 8}, + discovered: gpu.GPUInfo{System: "h200_sxm", GPUsPerNode: 8, NodesWithGPUs: 8, TotalGPUs: 64, SelectedShapeGPUs: 64}, check: func(t *testing.T, result *unstructured.Unstructured) { expectV1beta2GPUValues(t, result, "h200_sxm", 32) }, }, { - name: "v1beta2 an empty hardware object is automatic discovery", - request: v1beta2DGDRForAdmission(func(spec map[string]any) { - spec["hardware"] = map[string]any{} - }, nil), + name: "v1beta2 same-SKU nodes with different GPU counts use the selected node shape", + request: v1beta2DGDRForAdmission(nil, nil), gpuDiscovery: true, - discovered: gpu.GPUInfo{System: "h200_sxm", GPUsPerNode: 8, NodesWithGPUs: 1}, + discovered: gpu.GPUInfo{System: "h200_sxm", GPUsPerNode: 8, NodesWithGPUs: 2, TotalGPUs: 12, SelectedShapeGPUs: 8}, check: func(t *testing.T, result *unstructured.Unstructured) { expectV1beta2GPUValues(t, result, "h200_sxm", 8) }, }, + { + name: "v1beta2 empty typed hardware leaves native parameters authoritative", + request: v1beta2DGDRForAdmission(func(spec map[string]any) { + spec["hardware"] = map[string]any{} + setV1beta2SearchSpace(spec, map[string]any{"hardware_sku": "future_accelerator", "gpu_budget": int64(8)}) + }, nil), + gpuDiscovery: true, + discoveryErr: errors.New("the cluster must not be read"), + check: expectV1beta2HardwareOptOut, + }, { name: "v1beta2 a SKU alone keeps the SKU and persists the capped budget", request: v1beta2DGDRForAdmission(func(spec map[string]any) { spec["hardware"] = map[string]any{"gpu": map[string]any{"sku": "h100_sxm"}} }, nil), gpuDiscovery: true, - discovered: gpu.GPUInfo{System: "h100_sxm", GPUsPerNode: 8, NodesWithGPUs: 8}, + discovered: gpu.GPUInfo{System: "h100_sxm", GPUsPerNode: 8, NodesWithGPUs: 8, TotalGPUs: 64, SelectedShapeGPUs: 64}, check: func(t *testing.T, result *unstructured.Unstructured) { expectV1beta2GPUValues(t, result, "h100_sxm", 32) }, }, + { + name: "v1beta2 explicit node width wins while admission discovers a missing budget", + request: v1beta2DGDRForAdmission(func(spec map[string]any) { + spec["hardware"] = map[string]any{ + "gpu": map[string]any{"sku": "h200_sxm"}, + "numGpusPerNode": int64(4), + } + }, nil), + gpuDiscovery: true, + discovered: discoveredH200, + check: func(t *testing.T, result *unstructured.Unstructured) { + expectV1beta2GPUValues(t, result, "h200_sxm", 8, 4) + }, + }, { name: "v1beta2 a budget alone keeps the budget and discovers the SKU", request: v1beta2DGDRForAdmission(func(spec map[string]any) { @@ -832,14 +860,39 @@ func TestDynamoGraphDeploymentRequestValidator_Validate(t *testing.T) { }, }, { - name: "v1beta2 a SKU and a budget are kept without reading the cluster", + name: "v1beta2 explicit SKU and budget discover the missing node width", request: v1beta2DGDRForAdmission(func(spec map[string]any) { - spec["hardware"] = map[string]any{"gpu": map[string]any{"sku": "h100_sxm", "budget": int64(4)}} + spec["hardware"] = map[string]any{ + "gpu": map[string]any{"sku": "h200_sxm", "budget": int64(4)}, + } + }, nil), + gpuDiscovery: true, + discovered: discoveredH200, + check: func(t *testing.T, result *unstructured.Unstructured) { + expectV1beta2GPUValues(t, result, "h200_sxm", 4, 8) + }, + }, + { + name: "v1beta2 complete hardware is kept without reading the cluster", + request: v1beta2DGDRForAdmission(func(spec map[string]any) { + spec["hardware"] = map[string]any{"gpu": map[string]any{"sku": "h100_sxm", "budget": int64(4)}, "numGpusPerNode": int64(4)} }, nil), gpuDiscovery: true, discoveryErr: errors.New("the cluster must not be read"), check: func(t *testing.T, result *unstructured.Unstructured) { - expectV1beta2GPUValues(t, result, "h100_sxm", 4) + expectV1beta2GPUValues(t, result, "h100_sxm", 4, 4) + }, + }, + { + name: "v1beta2 unknown SKU without capacity or topology reports the manual override", + request: v1beta2DGDRForAdmission(func(spec map[string]any) { + spec["hardware"] = map[string]any{ + "gpu": map[string]any{"sku": "future_accelerator"}, + } + }, nil), + gpuDiscovery: true, + wantWebhook: []string{ + `spec.hardware.gpu.budget: Required value: cannot discover GPU capacity or topology for SKU "future_accelerator" because it is not known to the bundled AISimulate catalog; set spec.hardware.gpu.budget and spec.hardware.numGpusPerNode explicitly`, }, }, { @@ -860,7 +913,7 @@ func TestDynamoGraphDeploymentRequestValidator_Validate(t *testing.T) { discoveryErr: errors.New("no GPUs found"), wantWebhook: []string{ "spec.hardware.gpu: Required value: the cluster's GPUs could not be discovered (no GPUs found); " + - "set spec.hardware.gpu.sku and spec.hardware.gpu.budget", + "set spec.hardware.gpu.sku, spec.hardware.gpu.budget, and spec.hardware.numGpusPerNode", }, }, { @@ -891,7 +944,7 @@ func TestDynamoGraphDeploymentRequestValidator_Validate(t *testing.T) { spec["hardware"] = map[string]any{"gpu": map[string]any{"sku": "h200_sxm"}} }, nil), gpuDiscovery: true, - discovered: gpu.GPUInfo{System: "h200_sxm", GPUsPerNode: 8, NodesWithGPUs: 1}, + discovered: gpu.GPUInfo{System: "h200_sxm", GPUsPerNode: 8, NodesWithGPUs: 1, TotalGPUs: 8, SelectedShapeGPUs: 8}, check: func(t *testing.T, result *unstructured.Unstructured) { expectV1beta2GPUValues(t, result, "h200_sxm", 8) }, @@ -938,7 +991,7 @@ func TestDynamoGraphDeploymentRequestValidator_Validate(t *testing.T) { name: "v1beta2 explicit values are all kept", request: v1beta2DGDRForAdmission(func(spec map[string]any) { spec["backends"] = []any{"vllm"} - spec["hardware"] = map[string]any{"gpu": map[string]any{"sku": "h100_sxm", "budget": int64(4)}} + spec["hardware"] = map[string]any{"gpu": map[string]any{"sku": "h100_sxm", "budget": int64(4)}, "numGpusPerNode": int64(4)} spec["workload"] = map[string]any{"static": map[string]any{"isl": int64(10)}} spec["objective"] = map[string]any{"mode": "Optimize", "metric": "TTFT"} spec["aisimulate"] = map[string]any{ @@ -1452,6 +1505,7 @@ func expectV1beta2ExplicitValues(t *testing.T, obj *unstructured.Unstructured) { value float64 }{ {[]string{"spec", "hardware", "gpu", "budget"}, 4}, + {[]string{"spec", "hardware", "numGpusPerNode"}, 4}, {[]string{"spec", "workload", "static", "isl"}, 10}, {[]string{"spec", "recommendation", "maxCandidates"}, 3}, } { @@ -1496,6 +1550,20 @@ func expectV1beta2AdmissionDefaults(t *testing.T, obj *unstructured.Unstructured if got := v1beta2Number(t, obj, "spec", "hardware", "gpu", "budget"); got != 8 { t.Fatalf("hardware.gpu.budget = %v, want 8", got) } + if got := v1beta2Number(t, obj, "spec", "hardware", "numGpusPerNode"); got != 8 { + t.Fatalf("hardware.numGpusPerNode = %v, want 8", got) + } +} + +func expectV1beta2HardwareOptOut(t *testing.T, obj *unstructured.Unstructured) { + t.Helper() + hardware, found, err := unstructured.NestedMap(obj.Object, "spec", "hardware") + if err != nil || !found || len(hardware) != 0 { + t.Fatalf("hardware = %#v, found %v, err %v; want exact empty-object opt-out", hardware, found, err) + } + if got := v1beta2String(t, obj, "spec", "aisimulate", "parameters", "search_space", "hardware_sku"); got != "future_accelerator" { + t.Fatalf("parameters.search_space.hardware_sku = %q, want future_accelerator", got) + } } // setV1beta2SearchSpace sets spec.aisimulate.parameters.search_space to searchSpace, replacing @@ -1511,25 +1579,20 @@ func setV1beta2SearchSpace(spec map[string]any, searchSpace map[string]any) { // setBeta2AdmissionHardware names the hardware, so fixtures do not depend on GPU discovery. func setBeta2AdmissionHardware(request *nvidiacomv1beta2.DynamoGraphDeploymentRequest) { - sku, budget := "h100_sxm", int32(4) + sku, budget, width := "h100_sxm", int32(4), int32(4) request.Spec.Hardware = &nvidiacomv1beta2.HardwareSpec{ - GPU: &nvidiacomv1beta2.GPUHardwareSpec{SKU: &sku, Budget: &budget}, + GPU: &nvidiacomv1beta2.GPUHardwareSpec{SKU: &sku, Budget: &budget}, + NumGPUsPerNode: &width, } } // expectV1beta2GPU asserts the typed GPU hardware the API server stored. -func expectV1beta2GPU(t *testing.T, result *unstructured.Unstructured, sku string, budget float64) { - t.Helper() - if got := v1beta2String(t, result, "spec", "hardware", "gpu", "sku"); got != sku { - t.Fatalf("hardware.gpu.sku = %q, want %q", got, sku) - } - if got := v1beta2Number(t, result, "spec", "hardware", "gpu", "budget"); got != budget { - t.Fatalf("hardware.gpu.budget = %v, want %v", got, budget) - } +func expectV1beta2GPU(t *testing.T, result *unstructured.Unstructured, sku string, budget float64, width ...float64) { + expectV1beta2GPUValues(t, result, sku, budget, width...) } // expectV1beta2GPUValues checks the typed hardware a Request was stored with. -func expectV1beta2GPUValues(t *testing.T, obj *unstructured.Unstructured, sku string, budget float64) { +func expectV1beta2GPUValues(t *testing.T, obj *unstructured.Unstructured, sku string, budget float64, width ...float64) { t.Helper() if got := v1beta2String(t, obj, "spec", "hardware", "gpu", "sku"); got != sku { t.Fatalf("hardware.gpu.sku = %q, want %q", got, sku) @@ -1537,4 +1600,11 @@ func expectV1beta2GPUValues(t *testing.T, obj *unstructured.Unstructured, sku st if got := v1beta2Number(t, obj, "spec", "hardware", "gpu", "budget"); got != budget { t.Fatalf("hardware.gpu.budget = %v, want %v", got, budget) } + wantWidth := float64(8) + if len(width) > 0 { + wantWidth = width[0] + } + if got := v1beta2Number(t, obj, "spec", "hardware", "numGpusPerNode"); got != wantWidth { + t.Fatalf("hardware.numGpusPerNode = %v, want %v", got, wantWidth) + } } diff --git a/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md b/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md index efe88648072f..81eb62558e42 100644 --- a/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md +++ b/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md @@ -3437,7 +3437,7 @@ _Appears in:_ | --- | --- | --- | --- | | `model` _[ModelSpec](#modelspec)_ | Model identifies the model whose deployment configurations are evaluated. | | Optional: \{\}
| | `backends` _[Backend](#backend) array_ | Backends lists one or more inference backends searched by AISimulate and used by generated candidates. | | Enum: [vllm sglang trtllm]
MinItems: 1
Optional: \{\}
| -| `hardware` _[HardwareSpec](#v1beta2-hardwarespec)_ | Hardware bounds the accelerator configurations evaluated by the search.
Admission discovers any omitted hardware values from the cluster. | | Optional: \{\}
| +| `hardware` _[HardwareSpec](#v1beta2-hardwarespec)_ | Hardware bounds the accelerator configurations evaluated by the search.
When omitted, admission discovers SKU, budget and node width. An exactly empty object opts
out of operator hardware discovery and typed defaults. Admission completes every nonempty
object with any missing hardware values; explicit values remain authoritative. | | Optional: \{\}
| | `workload` _[WorkloadSpec](#v1beta2-workloadspec)_ | Workload defines the synthetic traffic model used for every candidate evaluation.
When omitted, admission supplies a static shape of ISL 1024, OSL 128 and concurrency 10;
an explicit empty workload receives the same shape from admission. An explicit
workload.static: \{\} opts out of that shape, leaving the workload to
spec.aisimulate.parameters.workload. spec.aisimulate.parameters.workload overrides the
fields it names.
The default evaluates a fixed concurrency, so it creates neither sweep.max_trials nor a
KV-load ratio search. | | Optional: \{\}
| | `objective` _[ObjectiveSpec](#objectivespec)_ | Objective defines scalar optimization or a multi-objective Pareto search.
When omitted, admission supplies the object. Its mode defaults to Optimize, and admission
supplies the conditional scalar metric. | | Optional: \{\}
| | `aisimulate` _[AISimulateSpec](#aisimulatespec)_ | AISimulate selects the execution artifact and supplies native AISimulate input. | | Optional: \{\}
| @@ -3479,7 +3479,7 @@ _Appears in:_ | --- | --- | --- | --- | | `model` _[ModelSpec](#modelspec)_ | Model identifies the model whose deployment configurations are evaluated. | | Optional: \{\}
| | `backends` _[Backend](#backend) array_ | Backends lists one or more inference backends searched by AISimulate and used by generated candidates. | | Enum: [vllm sglang trtllm]
MinItems: 1
Optional: \{\}
| -| `hardware` _[HardwareSpec](#v1beta2-hardwarespec)_ | Hardware bounds the accelerator configurations evaluated by the search.
Admission discovers any omitted hardware values from the cluster. | | Optional: \{\}
| +| `hardware` _[HardwareSpec](#v1beta2-hardwarespec)_ | Hardware bounds the accelerator configurations evaluated by the search.
When omitted, admission discovers SKU, budget and node width. An exactly empty object opts
out of operator hardware discovery and typed defaults. Admission completes every nonempty
object with any missing hardware values; explicit values remain authoritative. | | Optional: \{\}
| | `workload` _[WorkloadSpec](#v1beta2-workloadspec)_ | Workload defines the synthetic traffic model used for every candidate evaluation.
When omitted, admission supplies a static shape of ISL 1024, OSL 128 and concurrency 10;
an explicit empty workload receives the same shape from admission. An explicit
workload.static: \{\} opts out of that shape, leaving the workload to
spec.aisimulate.parameters.workload. spec.aisimulate.parameters.workload overrides the
fields it names.
The default evaluates a fixed concurrency, so it creates neither sweep.max_trials nor a
KV-load ratio search. | | Optional: \{\}
| | `objective` _[ObjectiveSpec](#objectivespec)_ | Objective defines scalar optimization or a multi-objective Pareto search.
When omitted, admission supplies the object. Its mode defaults to Optimize, and admission
supplies the conditional scalar metric. | | Optional: \{\}
| | `aisimulate` _[AISimulateSpec](#aisimulatespec)_ | AISimulate selects the execution artifact and supplies native AISimulate input. | | Optional: \{\}
| @@ -3542,7 +3542,7 @@ _Appears in:_ | --- | --- | --- | --- | | `model` _[ModelSpec](#modelspec)_ | Model identifies the model whose deployment configurations are evaluated. | | Optional: \{\}
| | `backends` _[Backend](#backend) array_ | Backends lists one or more inference backends searched by AISimulate and used by generated candidates. | | Enum: [vllm sglang trtllm]
MinItems: 1
Optional: \{\}
| -| `hardware` _[HardwareSpec](#v1beta2-hardwarespec)_ | Hardware bounds the accelerator configurations evaluated by the search.
Admission discovers any omitted hardware values from the cluster. | | Optional: \{\}
| +| `hardware` _[HardwareSpec](#v1beta2-hardwarespec)_ | Hardware bounds the accelerator configurations evaluated by the search.
When omitted, admission discovers SKU, budget and node width. An exactly empty object opts
out of operator hardware discovery and typed defaults. Admission completes every nonempty
object with any missing hardware values; explicit values remain authoritative. | | Optional: \{\}
| | `workload` _[WorkloadSpec](#v1beta2-workloadspec)_ | Workload defines the synthetic traffic model used for every candidate evaluation.
When omitted, admission supplies a static shape of ISL 1024, OSL 128 and concurrency 10;
an explicit empty workload receives the same shape from admission. An explicit
workload.static: \{\} opts out of that shape, leaving the workload to
spec.aisimulate.parameters.workload. spec.aisimulate.parameters.workload overrides the
fields it names.
The default evaluates a fixed concurrency, so it creates neither sweep.max_trials nor a
KV-load ratio search. | | Optional: \{\}
| | `objective` _[ObjectiveSpec](#objectivespec)_ | Objective defines scalar optimization or a multi-objective Pareto search.
When omitted, admission supplies the object. Its mode defaults to Optimize, and admission
supplies the conditional scalar metric. | | Optional: \{\}
| | `aisimulate` _[AISimulateSpec](#aisimulatespec)_ | AISimulate selects the execution artifact and supplies native AISimulate input. | | Optional: \{\}
| @@ -3607,15 +3607,16 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | -| `sku` _string_ | SKU identifies the accelerator using the AISimulate hardware catalog name.
When omitted, admission discovers the accelerator from the cluster. | | MinLength: 1
Optional: \{\}
| -| `budget` _integer_ | Budget is the maximum total number of GPUs one candidate may use.
When omitted, admission derives the available GPU budget from the cluster. | | Minimum: 1
Optional: \{\}
| +| `sku` _string_ | SKU identifies the accelerator using the AISimulate hardware catalog name.
Known values in the bundled AISimulate release are `a100_pcie`, `a100_sxm`,
`a30`, `b200_sxm`, `b300_sxm`, `b60`, `gb200`, `gb300`, `h100_pcie`,
`h100_sxm`, `h200_sxm`, `l4`, `l40s`, and `rtx_pro_6000_server`.
The field remains open so an external AISimulate systems catalog can add identifiers.
Admission passes an unknown identifier through unchanged and warns when `budget` and the
parent `numGpusPerNode` are also explicit. Without both, it rejects the identifier because
cluster discovery cannot associate capacity or topology with it. When omitted from a
nonempty hardware object, admission discovers the accelerator. | | MinLength: 1
Optional: \{\}
| +| `budget` _integer_ | Budget is the maximum total number of GPUs one candidate may use.
When omitted for a known SKU, admission derives the available GPU budget from the cluster.
An SKU unknown to the bundled AISimulate catalog requires an explicit `budget` and an
explicit `numGpusPerNode` in the parent hardware object. | | Minimum: 1
Optional: \{\}
| #### v1beta2 HardwareSpec -HardwareSpec bounds the accelerator configurations evaluated by the search. +HardwareSpec bounds the accelerator configurations evaluated by the search. An exactly empty +object opts out of operator cluster discovery and typed hardware defaults. @@ -3627,7 +3628,8 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | -| `gpu` _[GPUHardwareSpec](#gpuhardwarespec)_ | GPU describes accelerator identity and the maximum allocation per candidate.
Omitted fields are resolved independently by admission. | | Optional: \{\}
| +| `gpu` _[GPUHardwareSpec](#gpuhardwarespec)_ | GPU describes accelerator identity and the maximum allocation per candidate.
In a nonempty hardware object, admission resolves an omitted SKU or budget from the cluster. | | Optional: \{\}
| +| `numGpusPerNode` _integer_ | NumGPUsPerNode is the number of GPUs available to one candidate on each node. It lowers
to AISimulate's num_gpus_per_node input.
An explicit value is authoritative. In a nonempty hardware object, admission discovers
an omitted value from the selected cluster node shape. The Run controller never discovers
this value. An exactly empty hardware object opts out of all operator hardware discovery
and typed hardware defaults, leaving AISimulate input and defaults authoritative. | | Minimum: 1
Optional: \{\}
| #### KVLoadRatio @@ -3685,7 +3687,7 @@ _Appears in:_ | --- | --- | --- | --- | | `model` _[ModelSpec](#modelspec)_ | Model identifies the model whose deployment configurations are evaluated. | | Optional: \{\}
| | `backends` _[Backend](#backend) array_ | Backends lists one or more inference backends searched by AISimulate and used by generated candidates. | | Enum: [vllm sglang trtllm]
MinItems: 1
Optional: \{\}
| -| `hardware` _[HardwareSpec](#v1beta2-hardwarespec)_ | Hardware bounds the accelerator configurations evaluated by the search.
Admission discovers any omitted hardware values from the cluster. | | Optional: \{\}
| +| `hardware` _[HardwareSpec](#v1beta2-hardwarespec)_ | Hardware bounds the accelerator configurations evaluated by the search.
When omitted, admission discovers SKU, budget and node width. An exactly empty object opts
out of operator hardware discovery and typed defaults. Admission completes every nonempty
object with any missing hardware values; explicit values remain authoritative. | | Optional: \{\}
| | `workload` _[WorkloadSpec](#v1beta2-workloadspec)_ | Workload defines the synthetic traffic model used for every candidate evaluation.
When omitted, admission supplies a static shape of ISL 1024, OSL 128 and concurrency 10;
an explicit empty workload receives the same shape from admission. An explicit
workload.static: \{\} opts out of that shape, leaving the workload to
spec.aisimulate.parameters.workload. spec.aisimulate.parameters.workload overrides the
fields it names.
The default evaluates a fixed concurrency, so it creates neither sweep.max_trials nor a
KV-load ratio search. | | Optional: \{\}
| | `objective` _[ObjectiveSpec](#objectivespec)_ | Objective defines scalar optimization or a multi-objective Pareto search.
When omitted, admission supplies the object. Its mode defaults to Optimize, and admission
supplies the conditional scalar metric. | | Optional: \{\}
| | `aisimulate` _[AISimulateSpec](#aisimulatespec)_ | AISimulate selects the execution artifact and supplies native AISimulate input. | | Optional: \{\}
| diff --git a/docs/fern/pages/reference/kubernetes-api/dynamo-graph-deployment-request.mdx b/docs/fern/pages/reference/kubernetes-api/dynamo-graph-deployment-request.mdx index a54219ce6353..cfd7b92adca3 100644 --- a/docs/fern/pages/reference/kubernetes-api/dynamo-graph-deployment-request.mdx +++ b/docs/fern/pages/reference/kubernetes-api/dynamo-graph-deployment-request.mdx @@ -21,6 +21,61 @@ and modify a request through the API version that owns its native fields; do not set or edit the conversion field directly. An unchanged conversion view can be written back without changing the stored request. +### v1beta2 hardware SKU + +`spec.hardware.gpu.sku` in `nvidia.com/v1beta2` is an open string containing an +AISimulate system-catalog identifier. The AISimulate release bundled with this +Dynamo release includes `a100_pcie`, `a100_sxm`, `a30`, `b200_sxm`, `b300_sxm`, +`b60`, `gb200`, `gb300`, `h100_pcie`, `h100_sxm`, `h200_sxm`, `l4`, `l40s`, and +`rtx_pro_6000_server`. + +Admission passes an explicit SKU through unchanged. It warns when the bundled +catalog does not contain that identifier, but does not reject it: an external +AISimulate systems catalog may provide additional identifiers. Set `sku`, +`budget`, and `numGpusPerNode` to override cluster hardware discovery completely. +With an incomplete unknown SKU, admission cannot associate discovered capacity +or topology with that identifier and rejects the request with instructions to set +the missing values explicitly. + +When `spec.hardware` is omitted, admission discovers all three values from the +cluster. An exactly empty `hardware: {}` opts out of discovery and all typed +hardware defaults, leaving `spec.aisimulate.parameters` authoritative. Any +nonempty partial object is completed from discovery. Explicit values are kept. + +```yaml +apiVersion: nvidia.com/v1beta2 +kind: DynamoGraphDeploymentRequest +metadata: + name: future-accelerator +spec: + model: + name: Qwen/Qwen3-0.6B + hardware: + gpu: + sku: future_accelerator # Passed to AISimulate unchanged; admission warns if unknown. + budget: 8 + numGpusPerNode: 8 # Together, the three values bypass cluster discovery. +``` + +To supply hardware only through native AISimulate parameters, opt out of typed +discovery explicitly: + +```yaml +apiVersion: nvidia.com/v1beta2 +kind: DynamoGraphDeploymentRequest +metadata: + name: native-hardware-input +spec: + model: + name: Qwen/Qwen3-0.6B + hardware: {} + aisimulate: + parameters: + search_space: + hardware_sku: h200_sxm + gpu_budget: 8 +``` + ## Spec reference Only `model` is unconditionally required by the schema. Admission rejects a new DGDR whose supplied `spec.image` has no parseable semantic-version tag unless `spec.runtimeVersionOverride` is set. When `spec.image` is omitted, the operator supplies a versioned default as described below. diff --git a/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx b/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx index 3b04c19356f4..ca8f48d0acb4 100644 --- a/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx +++ b/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx @@ -4800,7 +4800,7 @@ See [Backend](#backend). -Hardware bounds the accelerator configurations evaluated by the search. Admission discovers any omitted hardware values from the cluster. +Hardware bounds the accelerator configurations evaluated by the search. When omitted, admission discovers SKU, budget and node width. An exactly empty object opts out of operator hardware discovery and typed defaults. Admission completes every nonempty object with any missing hardware values; explicit values remain authoritative. See [HardwareSpec](#v1beta2-hardwarespec). **Validation:** Optional: \{\} @@ -4882,7 +4882,7 @@ See [Backend](#backend). -Hardware bounds the accelerator configurations evaluated by the search. Admission discovers any omitted hardware values from the cluster. +Hardware bounds the accelerator configurations evaluated by the search. When omitted, admission discovers SKU, budget and node width. An exactly empty object opts out of operator hardware discovery and typed defaults. Admission completes every nonempty object with any missing hardware values; explicit values remain authoritative. See [HardwareSpec](#v1beta2-hardwarespec). **Validation:** Optional: \{\} @@ -5004,7 +5004,7 @@ See [Backend](#backend). -Hardware bounds the accelerator configurations evaluated by the search. Admission discovers any omitted hardware values from the cluster. +Hardware bounds the accelerator configurations evaluated by the search. When omitted, admission discovers SKU, budget and node width. An exactly empty object opts out of operator hardware discovery and typed defaults. Admission completes every nonempty object with any missing hardware values; explicit values remain authoritative. See [HardwareSpec](#v1beta2-hardwarespec). **Validation:** Optional: \{\} @@ -5134,28 +5134,34 @@ GPUHardwareSpec describes accelerator identity and the maximum allocation per ca **Appears in:** [HardwareSpec](#v1beta2-hardwarespec) -SKU identifies the accelerator using the AISimulate hardware catalog name. When omitted, admission discovers the accelerator from the cluster. +SKU identifies the accelerator using the AISimulate hardware catalog name. Known values in the bundled AISimulate release are `a100_pcie`, `a100_sxm`, `a30`, `b200_sxm`, `b300_sxm`, `b60`, `gb200`, `gb300`, `h100_pcie`, `h100_sxm`, `h200_sxm`, `l4`, `l40s`, and `rtx_pro_6000_server`. The field remains open so an external AISimulate systems catalog can add identifiers. Admission passes an unknown identifier through unchanged and warns when `budget` and the parent `numGpusPerNode` are also explicit. Without both, it rejects the identifier because cluster discovery cannot associate capacity or topology with it. When omitted from a nonempty hardware object, admission discovers the accelerator. **Validation:** MinLength: 1 Optional: \{\} -Budget is the maximum total number of GPUs one candidate may use. When omitted, admission derives the available GPU budget from the cluster. +Budget is the maximum total number of GPUs one candidate may use. When omitted for a known SKU, admission derives the available GPU budget from the cluster. An SKU unknown to the bundled AISimulate catalog requires an explicit `budget` and an explicit `numGpusPerNode` in the parent hardware object. **Validation:** Minimum: 1 Optional: \{\} -HardwareSpec bounds the accelerator configurations evaluated by the search. +HardwareSpec bounds the accelerator configurations evaluated by the search. An exactly empty +object opts out of operator cluster discovery and typed hardware defaults. **Kind:** `type` **Appears in:** [DynamoGraphDeploymentRequestNativeSpec](#v1beta2-dynamographdeploymentrequestnativespec), [DynamoGraphDeploymentRequestSpec](#v1beta2-dynamographdeploymentrequestspec), [DynamoGraphDeploymentRunSpec](#dynamographdeploymentrunspec), [NativeSearchInput](#nativesearchinput) -GPU describes accelerator identity and the maximum allocation per candidate. Omitted fields are resolved independently by admission. +GPU describes accelerator identity and the maximum allocation per candidate. In a nonempty hardware object, admission resolves an omitted SKU or budget from the cluster. See [GPUHardwareSpec](#gpuhardwarespec). **Validation:** Optional: \{\} + + +NumGPUsPerNode is the number of GPUs available to one candidate on each node. It lowers to AISimulate's num_gpus_per_node input. An explicit value is authoritative. In a nonempty hardware object, admission discovers an omitted value from the selected cluster node shape. The Run controller never discovers this value. An exactly empty hardware object opts out of all operator hardware discovery and typed hardware defaults, leaving AISimulate input and defaults authoritative. +**Validation:** Minimum: 1 Optional: \{\} + @@ -5203,7 +5209,7 @@ See [Backend](#backend). -Hardware bounds the accelerator configurations evaluated by the search. Admission discovers any omitted hardware values from the cluster. +Hardware bounds the accelerator configurations evaluated by the search. When omitted, admission discovers SKU, budget and node width. An exactly empty object opts out of operator hardware discovery and typed defaults. Admission completes every nonempty object with any missing hardware values; explicit values remain authoritative. See [HardwareSpec](#v1beta2-hardwarespec). **Validation:** Optional: \{\}