diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index f29fea96d..8e5463b53 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -40,7 +40,7 @@ import ( monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - runtime "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime" ) // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. diff --git a/bundle/manifests/amd-gpu-operator.clusterserviceversion.yaml b/bundle/manifests/amd-gpu-operator.clusterserviceversion.yaml index 3267e3158..295d65673 100644 --- a/bundle/manifests/amd-gpu-operator.clusterserviceversion.yaml +++ b/bundle/manifests/amd-gpu-operator.clusterserviceversion.yaml @@ -35,8 +35,8 @@ metadata: ] capabilities: Seamless Upgrades categories: AI/Machine Learning,Monitoring - containerImage: registry.test.pensando.io:5000/amd-gpu-operator:dev - createdAt: "2026-07-09T09:40:15Z" + containerImage: docker.io/rocm/amd-gpu-operator:dev + createdAt: "2026-09-23T02:56:40Z" description: |- Operator responsible for deploying AMD GPU kernel drivers, device plugin, device test runner and device metrics exporter For more information, visit [documentation](https://instinct.docs.amd.com/projects/gpu-operator/en/latest/) @@ -1476,6 +1476,13 @@ spec: - nodes verbs: - get + - apiGroups: + - kmm.sigs.x-k8s.io + resources: + - nodemodulesconfigs + verbs: + - get + - delete serviceAccountName: amd-gpu-operator-dra-driver - rules: - apiGroups: diff --git a/bundle/manifests/amd.com_deviceconfigs.yaml b/bundle/manifests/amd.com_deviceconfigs.yaml index cdbf95ccb..99c2a8cd6 100644 --- a/bundle/manifests/amd.com_deviceconfigs.yaml +++ b/bundle/manifests/amd.com_deviceconfigs.yaml @@ -147,9 +147,10 @@ spec: operator: description: |- Operator represents a key's relationship to the value. - Valid operators are Exists and Equal. Defaults to Equal. + Valid operators are Exists, Equal, Lt, and Gt. Defaults to Equal. Exists is equivalent to wildcard for value, so that a pod can tolerate all taints of a particular category. + Lt and Gt perform numeric comparisons (requires feature gate TaintTolerationComparisonOperators). type: string tolerationSeconds: description: |- @@ -261,9 +262,10 @@ spec: operator: description: |- Operator represents a key's relationship to the value. - Valid operators are Exists and Equal. Defaults to Equal. + Valid operators are Exists, Equal, Lt, and Gt. Defaults to Equal. Exists is equivalent to wildcard for value, so that a pod can tolerate all taints of a particular category. + Lt and Gt perform numeric comparisons (requires feature gate TaintTolerationComparisonOperators). type: string tolerationSeconds: description: |- @@ -354,9 +356,10 @@ spec: operator: description: |- Operator represents a key's relationship to the value. - Valid operators are Exists and Equal. Defaults to Equal. + Valid operators are Exists, Equal, Lt, and Gt. Defaults to Equal. Exists is equivalent to wildcard for value, so that a pod can tolerate all taints of a particular category. + Lt and Gt perform numeric comparisons (requires feature gate TaintTolerationComparisonOperators). type: string tolerationSeconds: description: |- @@ -457,9 +460,10 @@ spec: operator: description: |- Operator represents a key's relationship to the value. - Valid operators are Exists and Equal. Defaults to Equal. + Valid operators are Exists, Equal, Lt, and Gt. Defaults to Equal. Exists is equivalent to wildcard for value, so that a pod can tolerate all taints of a particular category. + Lt and Gt perform numeric comparisons (requires feature gate TaintTolerationComparisonOperators). type: string tolerationSeconds: description: |- @@ -719,9 +723,10 @@ spec: operator: description: |- Operator represents a key's relationship to the value. - Valid operators are Exists and Equal. Defaults to Equal. + Valid operators are Exists, Equal, Lt, and Gt. Defaults to Equal. Exists is equivalent to wildcard for value, so that a pod can tolerate all taints of a particular category. + Lt and Gt perform numeric comparisons (requires feature gate TaintTolerationComparisonOperators). type: string tolerationSeconds: description: |- @@ -1438,7 +1443,7 @@ spec: Claims lists the names of resources, defined in spec.resourceClaims, that are used by this container. - This is an alpha field and requires enabling the + This field depends on the DynamicResourceAllocation feature gate. This field is immutable. It can only be set for containers. @@ -1528,9 +1533,10 @@ spec: operator: description: |- Operator represents a key's relationship to the value. - Valid operators are Exists and Equal. Defaults to Equal. + Valid operators are Exists, Equal, Lt, and Gt. Defaults to Equal. Exists is equivalent to wildcard for value, so that a pod can tolerate all taints of a particular category. + Lt and Gt perform numeric comparisons (requires feature gate TaintTolerationComparisonOperators). type: string tolerationSeconds: description: |- @@ -1672,9 +1678,8 @@ spec: node. type: string timeAdded: - description: |- - TimeAdded represents the time at which the taint was added. - It is only written for NoExecute taints. + description: TimeAdded represents the time at which the + taint was added. format: date-time type: string value: @@ -1819,9 +1824,10 @@ spec: operator: description: |- Operator represents a key's relationship to the value. - Valid operators are Exists and Equal. Defaults to Equal. + Valid operators are Exists, Equal, Lt, and Gt. Defaults to Equal. Exists is equivalent to wildcard for value, so that a pod can tolerate all taints of a particular category. + Lt and Gt perform numeric comparisons (requires feature gate TaintTolerationComparisonOperators). type: string tolerationSeconds: description: |- diff --git a/config/crd/bases/amd.com_deviceconfigs.yaml b/config/crd/bases/amd.com_deviceconfigs.yaml index f98254d3f..a53844e71 100644 --- a/config/crd/bases/amd.com_deviceconfigs.yaml +++ b/config/crd/bases/amd.com_deviceconfigs.yaml @@ -143,9 +143,10 @@ spec: operator: description: |- Operator represents a key's relationship to the value. - Valid operators are Exists and Equal. Defaults to Equal. + Valid operators are Exists, Equal, Lt, and Gt. Defaults to Equal. Exists is equivalent to wildcard for value, so that a pod can tolerate all taints of a particular category. + Lt and Gt perform numeric comparisons (requires feature gate TaintTolerationComparisonOperators). type: string tolerationSeconds: description: |- @@ -257,9 +258,10 @@ spec: operator: description: |- Operator represents a key's relationship to the value. - Valid operators are Exists and Equal. Defaults to Equal. + Valid operators are Exists, Equal, Lt, and Gt. Defaults to Equal. Exists is equivalent to wildcard for value, so that a pod can tolerate all taints of a particular category. + Lt and Gt perform numeric comparisons (requires feature gate TaintTolerationComparisonOperators). type: string tolerationSeconds: description: |- @@ -350,9 +352,10 @@ spec: operator: description: |- Operator represents a key's relationship to the value. - Valid operators are Exists and Equal. Defaults to Equal. + Valid operators are Exists, Equal, Lt, and Gt. Defaults to Equal. Exists is equivalent to wildcard for value, so that a pod can tolerate all taints of a particular category. + Lt and Gt perform numeric comparisons (requires feature gate TaintTolerationComparisonOperators). type: string tolerationSeconds: description: |- @@ -453,9 +456,10 @@ spec: operator: description: |- Operator represents a key's relationship to the value. - Valid operators are Exists and Equal. Defaults to Equal. + Valid operators are Exists, Equal, Lt, and Gt. Defaults to Equal. Exists is equivalent to wildcard for value, so that a pod can tolerate all taints of a particular category. + Lt and Gt perform numeric comparisons (requires feature gate TaintTolerationComparisonOperators). type: string tolerationSeconds: description: |- @@ -715,9 +719,10 @@ spec: operator: description: |- Operator represents a key's relationship to the value. - Valid operators are Exists and Equal. Defaults to Equal. + Valid operators are Exists, Equal, Lt, and Gt. Defaults to Equal. Exists is equivalent to wildcard for value, so that a pod can tolerate all taints of a particular category. + Lt and Gt perform numeric comparisons (requires feature gate TaintTolerationComparisonOperators). type: string tolerationSeconds: description: |- @@ -1434,7 +1439,7 @@ spec: Claims lists the names of resources, defined in spec.resourceClaims, that are used by this container. - This is an alpha field and requires enabling the + This field depends on the DynamicResourceAllocation feature gate. This field is immutable. It can only be set for containers. @@ -1524,9 +1529,10 @@ spec: operator: description: |- Operator represents a key's relationship to the value. - Valid operators are Exists and Equal. Defaults to Equal. + Valid operators are Exists, Equal, Lt, and Gt. Defaults to Equal. Exists is equivalent to wildcard for value, so that a pod can tolerate all taints of a particular category. + Lt and Gt perform numeric comparisons (requires feature gate TaintTolerationComparisonOperators). type: string tolerationSeconds: description: |- @@ -1668,9 +1674,8 @@ spec: node. type: string timeAdded: - description: |- - TimeAdded represents the time at which the taint was added. - It is only written for NoExecute taints. + description: TimeAdded represents the time at which the + taint was added. format: date-time type: string value: @@ -1815,9 +1820,10 @@ spec: operator: description: |- Operator represents a key's relationship to the value. - Valid operators are Exists and Equal. Defaults to Equal. + Valid operators are Exists, Equal, Lt, and Gt. Defaults to Equal. Exists is equivalent to wildcard for value, so that a pod can tolerate all taints of a particular category. + Lt and Gt perform numeric comparisons (requires feature gate TaintTolerationComparisonOperators). type: string tolerationSeconds: description: |- diff --git a/config/rbac/dra_driver_cluster_role.yaml b/config/rbac/dra_driver_cluster_role.yaml index 5f2069856..bd3a230c1 100644 --- a/config/rbac/dra_driver_cluster_role.yaml +++ b/config/rbac/dra_driver_cluster_role.yaml @@ -35,3 +35,10 @@ rules: - nodes verbs: - get +- apiGroups: + - kmm.sigs.x-k8s.io + resources: + - nodemodulesconfigs + verbs: + - get + - delete diff --git a/hack/k8s-patch/metadata-patch/values.yaml b/hack/k8s-patch/metadata-patch/values.yaml index 1f676fa3e..c6992bed6 100644 --- a/hack/k8s-patch/metadata-patch/values.yaml +++ b/hack/k8s-patch/metadata-patch/values.yaml @@ -365,6 +365,17 @@ draDriver: deviceClass: # -- Create the gpu.amd.com DeviceClass resource. Set to false if managing the DRA driver independently. create: true + # -- Maps classic extended-resource requests (e.g. amd.com/gpu) to a DeviceClass. + # Set "" to omit the field. When autoPartition is true, this is applied to a + # second class, gpu.amd.com-spx, constrained to whole-GPU (spx) devices only, so + # classic requests can't resolve to a leftover partition fraction. When + # autoPartition is false, it stays on the single gpu.amd.com class as before. + extendedResourceName: amd.com/gpu + # -- Set to true when DeviceConfig.spec.draDriver.cmdLineArguments enables the + # DRA driver's AutoPartition feature gate on the matching nodes. Must be kept in + # sync with that feature-gates value; it only controls this chart's DeviceClass + # rendering and cannot detect the DeviceConfig's setting automatically. + autoPartition: false kmmModuleLoader: serviceAccount: annotations: {} diff --git a/hack/k8s-patch/template-patch/dra-driver-deviceclass.yaml b/hack/k8s-patch/template-patch/dra-driver-deviceclass.yaml index d4a7e6094..2ef51677f 100644 --- a/hack/k8s-patch/template-patch/dra-driver-deviceclass.yaml +++ b/hack/k8s-patch/template-patch/dra-driver-deviceclass.yaml @@ -12,4 +12,36 @@ spec: selectors: - cel: expression: "device.driver == 'gpu.amd.com'" +{{- if and .Values.draDriver.deviceClass.extendedResourceName (not .Values.draDriver.deviceClass.autoPartition) }} + extendedResourceName: {{ .Values.draDriver.deviceClass.extendedResourceName }} +{{- end }} +{{- if .Values.draDriver.deviceClass.autoPartition }} +--- +# When AutoPartition is enabled on the DRA driver, gpu.amd.com's device pool +# includes fractional partition devices (cpx/dpx/qpx) alongside whole GPUs (spx), +# all matched by the same selector above. extendedResourceName moves to this +# second, spx-constrained class instead of staying on the general one, so a +# classic amd.com/gpu request (which carries no selector of its own -- it's +# translated by the apiserver from resources.limits, not authored as a +# ResourceClaim) can only ever resolve to a whole GPU, never a partition +# fraction left over from another claim. +apiVersion: {{ $draApi }} +kind: DeviceClass +metadata: + name: gpu.amd.com-spx + labels: + app.kubernetes.io/component: amd-gpu + app.kubernetes.io/part-of: amd-gpu + {{- include "helm-charts-k8s.labels" . | nindent 4 }} +spec: + selectors: + - cel: + expression: >- + device.driver == 'gpu.amd.com' && + (!has(device.attributes["gpu.amd.com"].computePartition) || + device.attributes["gpu.amd.com"].computePartition == "spx") + {{- with .Values.draDriver.deviceClass.extendedResourceName }} + extendedResourceName: {{ . }} + {{- end }} +{{- end }} {{- end }} diff --git a/hack/k8s-patch/template-patch/dra-driver-rbac.yaml b/hack/k8s-patch/template-patch/dra-driver-rbac.yaml index b0da170f2..54eac91b9 100644 --- a/hack/k8s-patch/template-patch/dra-driver-rbac.yaml +++ b/hack/k8s-patch/template-patch/dra-driver-rbac.yaml @@ -31,6 +31,13 @@ rules: - update - patch - delete +- apiGroups: + - kmm.sigs.x-k8s.io + resources: + - nodemodulesconfigs + verbs: + - get + - delete --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding diff --git a/helm-charts-k8s/Chart.lock b/helm-charts-k8s/Chart.lock index 526228990..e9b9f562a 100644 --- a/helm-charts-k8s/Chart.lock +++ b/helm-charts-k8s/Chart.lock @@ -9,4 +9,4 @@ dependencies: repository: file://./charts/remediation-crds version: v1.0.0 digest: sha256:0806f6b6d7aa21be77bf1c91e720ae3238338a16f107df450a53b02ef940db1b -generated: "2026-07-09T09:40:11.647230235Z" +generated: "2026-09-23T02:56:24.120766399Z" diff --git a/helm-charts-k8s/README.md b/helm-charts-k8s/README.md index 0a1258bee..e476bd4c8 100644 --- a/helm-charts-k8s/README.md +++ b/helm-charts-k8s/README.md @@ -6,7 +6,7 @@ For the most detailed and up-to-date documentation please visit our Instinct Doc ## Introduction -AMD GPU Operator simplifies the deployment and management of AMD Instinct GPU accelerators within Kubernetes clusters. This project enables seamless configuration and operation of GPU-accelerated workloads, including machine learning, Generative AI, and other GPU-intensive applications. +AMD GPU Operator simplifies the deployment and management of AMD Instinct™ and AMD Radeon™ GPU accelerators within Kubernetes clusters. This project enables seamless configuration and operation of GPU-accelerated workloads, including machine learning, Generative AI, and other GPU-intensive applications. ## Components @@ -280,7 +280,9 @@ Kubernetes: `>= 1.29.0-0` | deviceConfig.spec.testRunner.tolerations | list | `[]` | test runner tolerations | | deviceConfig.spec.testRunner.upgradePolicy.maxUnavailable | int | `1` | the maximum number of Pods that can be unavailable during the update process | | deviceConfig.spec.testRunner.upgradePolicy.upgradeStrategy | string | `"RollingUpdate"` | the type of daemonset upgrade, RollingUpdate or OnDelete | +| draDriver.deviceClass.autoPartition | bool | `false` | Set to true when DeviceConfig.spec.draDriver.cmdLineArguments enables the DRA driver's AutoPartition feature gate on the matching nodes. Must be kept in sync with that feature-gates value; it only controls this chart's DeviceClass rendering and cannot detect the DeviceConfig's setting automatically. | | draDriver.deviceClass.create | bool | `true` | Create the gpu.amd.com DeviceClass resource. Set to false if managing the DRA driver independently. | +| draDriver.deviceClass.extendedResourceName | string | `"amd.com/gpu"` | Maps classic extended-resource requests (e.g. amd.com/gpu) to a DeviceClass. Set "" to omit the field. When autoPartition is true, this is applied to a second class, gpu.amd.com-spx, constrained to whole-GPU (spx) devices only, so classic requests can't resolve to a leftover partition fraction. When autoPartition is false, it stays on the single gpu.amd.com class as before. | | installdefaultNFDRule | bool | `true` | Default NFD rule will detect amd gpu based on pci vendor ID | | kmm.enabled | bool | `true` | Set to true/false to enable/disable the installation of kernel module management (KMM) operator subchart | | kmm.watch | bool | `true` | Set to true/false to enable/disable GPU operator watching and using KMM resources | diff --git a/helm-charts-k8s/crds/deviceconfig-crd.yaml b/helm-charts-k8s/crds/deviceconfig-crd.yaml index c54566694..a303d0cc9 100644 --- a/helm-charts-k8s/crds/deviceconfig-crd.yaml +++ b/helm-charts-k8s/crds/deviceconfig-crd.yaml @@ -152,9 +152,10 @@ spec: operator: description: |- Operator represents a key's relationship to the value. - Valid operators are Exists and Equal. Defaults to Equal. + Valid operators are Exists, Equal, Lt, and Gt. Defaults to Equal. Exists is equivalent to wildcard for value, so that a pod can tolerate all taints of a particular category. + Lt and Gt perform numeric comparisons (requires feature gate TaintTolerationComparisonOperators). type: string tolerationSeconds: description: |- @@ -266,9 +267,10 @@ spec: operator: description: |- Operator represents a key's relationship to the value. - Valid operators are Exists and Equal. Defaults to Equal. + Valid operators are Exists, Equal, Lt, and Gt. Defaults to Equal. Exists is equivalent to wildcard for value, so that a pod can tolerate all taints of a particular category. + Lt and Gt perform numeric comparisons (requires feature gate TaintTolerationComparisonOperators). type: string tolerationSeconds: description: |- @@ -359,9 +361,10 @@ spec: operator: description: |- Operator represents a key's relationship to the value. - Valid operators are Exists and Equal. Defaults to Equal. + Valid operators are Exists, Equal, Lt, and Gt. Defaults to Equal. Exists is equivalent to wildcard for value, so that a pod can tolerate all taints of a particular category. + Lt and Gt perform numeric comparisons (requires feature gate TaintTolerationComparisonOperators). type: string tolerationSeconds: description: |- @@ -462,9 +465,10 @@ spec: operator: description: |- Operator represents a key's relationship to the value. - Valid operators are Exists and Equal. Defaults to Equal. + Valid operators are Exists, Equal, Lt, and Gt. Defaults to Equal. Exists is equivalent to wildcard for value, so that a pod can tolerate all taints of a particular category. + Lt and Gt perform numeric comparisons (requires feature gate TaintTolerationComparisonOperators). type: string tolerationSeconds: description: |- @@ -723,9 +727,10 @@ spec: operator: description: |- Operator represents a key's relationship to the value. - Valid operators are Exists and Equal. Defaults to Equal. + Valid operators are Exists, Equal, Lt, and Gt. Defaults to Equal. Exists is equivalent to wildcard for value, so that a pod can tolerate all taints of a particular category. + Lt and Gt perform numeric comparisons (requires feature gate TaintTolerationComparisonOperators). type: string tolerationSeconds: description: |- @@ -1440,7 +1445,7 @@ spec: Claims lists the names of resources, defined in spec.resourceClaims, that are used by this container. - This is an alpha field and requires enabling the + This field depends on the DynamicResourceAllocation feature gate. This field is immutable. It can only be set for containers. @@ -1530,9 +1535,10 @@ spec: operator: description: |- Operator represents a key's relationship to the value. - Valid operators are Exists and Equal. Defaults to Equal. + Valid operators are Exists, Equal, Lt, and Gt. Defaults to Equal. Exists is equivalent to wildcard for value, so that a pod can tolerate all taints of a particular category. + Lt and Gt perform numeric comparisons (requires feature gate TaintTolerationComparisonOperators). type: string tolerationSeconds: description: |- @@ -1673,9 +1679,8 @@ spec: description: Required. The taint key to be applied to a node. type: string timeAdded: - description: |- - TimeAdded represents the time at which the taint was added. - It is only written for NoExecute taints. + description: TimeAdded represents the time at which the taint + was added. format: date-time type: string value: @@ -1818,9 +1823,10 @@ spec: operator: description: |- Operator represents a key's relationship to the value. - Valid operators are Exists and Equal. Defaults to Equal. + Valid operators are Exists, Equal, Lt, and Gt. Defaults to Equal. Exists is equivalent to wildcard for value, so that a pod can tolerate all taints of a particular category. + Lt and Gt perform numeric comparisons (requires feature gate TaintTolerationComparisonOperators). type: string tolerationSeconds: description: |- diff --git a/helm-charts-k8s/templates/dra-driver-deviceclass.yaml b/helm-charts-k8s/templates/dra-driver-deviceclass.yaml index d4a7e6094..2ef51677f 100644 --- a/helm-charts-k8s/templates/dra-driver-deviceclass.yaml +++ b/helm-charts-k8s/templates/dra-driver-deviceclass.yaml @@ -12,4 +12,36 @@ spec: selectors: - cel: expression: "device.driver == 'gpu.amd.com'" +{{- if and .Values.draDriver.deviceClass.extendedResourceName (not .Values.draDriver.deviceClass.autoPartition) }} + extendedResourceName: {{ .Values.draDriver.deviceClass.extendedResourceName }} +{{- end }} +{{- if .Values.draDriver.deviceClass.autoPartition }} +--- +# When AutoPartition is enabled on the DRA driver, gpu.amd.com's device pool +# includes fractional partition devices (cpx/dpx/qpx) alongside whole GPUs (spx), +# all matched by the same selector above. extendedResourceName moves to this +# second, spx-constrained class instead of staying on the general one, so a +# classic amd.com/gpu request (which carries no selector of its own -- it's +# translated by the apiserver from resources.limits, not authored as a +# ResourceClaim) can only ever resolve to a whole GPU, never a partition +# fraction left over from another claim. +apiVersion: {{ $draApi }} +kind: DeviceClass +metadata: + name: gpu.amd.com-spx + labels: + app.kubernetes.io/component: amd-gpu + app.kubernetes.io/part-of: amd-gpu + {{- include "helm-charts-k8s.labels" . | nindent 4 }} +spec: + selectors: + - cel: + expression: >- + device.driver == 'gpu.amd.com' && + (!has(device.attributes["gpu.amd.com"].computePartition) || + device.attributes["gpu.amd.com"].computePartition == "spx") + {{- with .Values.draDriver.deviceClass.extendedResourceName }} + extendedResourceName: {{ . }} + {{- end }} +{{- end }} {{- end }} diff --git a/helm-charts-k8s/templates/dra-driver-rbac.yaml b/helm-charts-k8s/templates/dra-driver-rbac.yaml index b0da170f2..54eac91b9 100644 --- a/helm-charts-k8s/templates/dra-driver-rbac.yaml +++ b/helm-charts-k8s/templates/dra-driver-rbac.yaml @@ -31,6 +31,13 @@ rules: - update - patch - delete +- apiGroups: + - kmm.sigs.x-k8s.io + resources: + - nodemodulesconfigs + verbs: + - get + - delete --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding diff --git a/helm-charts-k8s/values.yaml b/helm-charts-k8s/values.yaml index 1f676fa3e..c6992bed6 100644 --- a/helm-charts-k8s/values.yaml +++ b/helm-charts-k8s/values.yaml @@ -365,6 +365,17 @@ draDriver: deviceClass: # -- Create the gpu.amd.com DeviceClass resource. Set to false if managing the DRA driver independently. create: true + # -- Maps classic extended-resource requests (e.g. amd.com/gpu) to a DeviceClass. + # Set "" to omit the field. When autoPartition is true, this is applied to a + # second class, gpu.amd.com-spx, constrained to whole-GPU (spx) devices only, so + # classic requests can't resolve to a leftover partition fraction. When + # autoPartition is false, it stays on the single gpu.amd.com class as before. + extendedResourceName: amd.com/gpu + # -- Set to true when DeviceConfig.spec.draDriver.cmdLineArguments enables the + # DRA driver's AutoPartition feature gate on the matching nodes. Must be kept in + # sync with that feature-gates value; it only controls this chart's DeviceClass + # rendering and cannot detect the DeviceConfig's setting automatically. + autoPartition: false kmmModuleLoader: serviceAccount: annotations: {} diff --git a/internal/client/mock_client.go b/internal/client/mock_client.go index ee86fdc91..0c8903d5f 100644 --- a/internal/client/mock_client.go +++ b/internal/client/mock_client.go @@ -59,6 +59,25 @@ func (m *MockClient) EXPECT() *MockClientMockRecorder { return m.recorder } +// Apply mocks base method. +func (m *MockClient) Apply(arg0 context.Context, arg1 runtime.ApplyConfiguration, arg2 ...client.ApplyOption) error { + m.ctrl.T.Helper() + varargs := []any{arg0, arg1} + for _, a := range arg2 { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "Apply", varargs...) + ret0, _ := ret[0].(error) + return ret0 +} + +// Apply indicates an expected call of Apply. +func (mr *MockClientMockRecorder) Apply(arg0, arg1 any, arg2 ...any) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]any{arg0, arg1}, arg2...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Apply", reflect.TypeOf((*MockClient)(nil).Apply), varargs...) +} + // Create mocks base method. func (m *MockClient) Create(arg0 context.Context, arg1 client.Object, arg2 ...client.CreateOption) error { m.ctrl.T.Helper() @@ -259,25 +278,6 @@ func (mr *MockClientMockRecorder) SubResource(arg0 any) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SubResource", reflect.TypeOf((*MockClient)(nil).SubResource), arg0) } -// Apply mocks base method. -func (m *MockClient) Apply(arg0 context.Context, arg1 runtime.ApplyConfiguration, arg2 ...client.ApplyOption) error { - m.ctrl.T.Helper() - varargs := []any{arg0, arg1} - for _, a := range arg2 { - varargs = append(varargs, a) - } - ret := m.ctrl.Call(m, "Apply", varargs...) - ret0, _ := ret[0].(error) - return ret0 -} - -// Apply indicates an expected call of Apply. -func (mr *MockClientMockRecorder) Apply(arg0, arg1 any, arg2 ...any) *gomock.Call { - mr.mock.ctrl.T.Helper() - varargs := append([]any{arg0, arg1}, arg2...) - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Apply", reflect.TypeOf((*MockClient)(nil).Apply), varargs...) -} - // Update mocks base method. func (m *MockClient) Update(arg0 context.Context, arg1 client.Object, arg2 ...client.UpdateOption) error { m.ctrl.T.Helper() @@ -320,6 +320,25 @@ func (m *MockStatusWriter) EXPECT() *MockStatusWriterMockRecorder { return m.recorder } +// Apply mocks base method. +func (m *MockStatusWriter) Apply(arg0 context.Context, arg1 runtime.ApplyConfiguration, arg2 ...client.SubResourceApplyOption) error { + m.ctrl.T.Helper() + varargs := []any{arg0, arg1} + for _, a := range arg2 { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "Apply", varargs...) + ret0, _ := ret[0].(error) + return ret0 +} + +// Apply indicates an expected call of Apply. +func (mr *MockStatusWriterMockRecorder) Apply(arg0, arg1 any, arg2 ...any) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]any{arg0, arg1}, arg2...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Apply", reflect.TypeOf((*MockStatusWriter)(nil).Apply), varargs...) +} + // Create mocks base method. func (m *MockStatusWriter) Create(arg0 context.Context, arg1, arg2 client.Object, arg3 ...client.SubResourceCreateOption) error { m.ctrl.T.Helper() diff --git a/internal/controllers/device_config_reconciler.go b/internal/controllers/device_config_reconciler.go index 841f7735d..4012146fc 100644 --- a/internal/controllers/device_config_reconciler.go +++ b/internal/controllers/device_config_reconciler.go @@ -83,6 +83,10 @@ const ( deviceConfigFinalizer = "amd.node.kubernetes.io/deviceconfig-finalizer" testRunnerNodeLabelPrefix = "testrunner.amd.com" deviceClassName = "gpu.amd.com" + // deviceClassExtendedResourceName maps classic amd.com/gpu (resources.limits) + // requests onto DRA devices. Mirrors the default of + // helm-charts-k8s' draDriver.deviceClass.extendedResourceName Helm value. + deviceClassExtendedResourceName = "amd.com/gpu" ) var draAPIVersionPriority = []string{"v1", "v1beta2", "v1beta1"} @@ -1271,40 +1275,69 @@ func (dcrh *deviceConfigReconcilerHelper) handleDeviceClass(ctx context.Context, return nil } - logger := log.FromContext(ctx) - apiVersion := discoverDRAAPIVersion() - logger.Info("Discovered DRA API version", "apiVersion", apiVersion) + log.FromContext(ctx).Info("Discovered DRA API version", "apiVersion", apiVersion) + + if !utils.ShouldUseAutoPartition(devConfig) { + return dcrh.applyDeviceClass(ctx, apiVersion, deviceClassName, + "device.driver == '"+deviceClassName+"'", deviceClassExtendedResourceName) + } + + // When AutoPartition is enabled, gpu.amd.com's device pool includes fractional + // partition devices (cpx/dpx/qpx) alongside whole GPUs (spx), all matched by + // the same selector. A classic amd.com/gpu request (no selector of its own -- + // it's translated by the apiserver from resources.limits, not authored as a + // ResourceClaim) must not be able to resolve to a partition fraction left over + // from another claim, so extendedResourceName moves to a second, + // spx-constrained class instead of staying on the general one. Mirrors + // helm-charts-k8s/templates/dra-driver-deviceclass.yaml. + if err := dcrh.applyDeviceClass(ctx, apiVersion, deviceClassName, + "device.driver == '"+deviceClassName+"'", ""); err != nil { + return err + } + spxExpr := "device.driver == '" + deviceClassName + "' && " + + "(!has(device.attributes[\"" + deviceClassName + "\"].computePartition) || " + + "device.attributes[\"" + deviceClassName + "\"].computePartition == \"spx\")" + return dcrh.applyDeviceClass(ctx, apiVersion, deviceClassName+"-spx", spxExpr, deviceClassExtendedResourceName) +} +// applyDeviceClass creates a DeviceClass named name with the given CEL selector +// expression, setting extendedResourceName when non-empty. A no-op if the +// DeviceClass already exists -- this does not reconcile drift on an existing one. +func (dcrh *deviceConfigReconcilerHelper) applyDeviceClass(ctx context.Context, apiVersion, name, selectorExpr, extendedResourceName string) error { dc := &unstructured.Unstructured{} dc.SetGroupVersionKind(schema.GroupVersionKind{ Group: "resource.k8s.io", Version: apiVersion, Kind: "DeviceClass", }) - dc.SetName(deviceClassName) + dc.SetName(name) dc.SetLabels(map[string]string{ "app.kubernetes.io/component": "amd-gpu", "app.kubernetes.io/part-of": "amd-gpu", }) - dc.Object["spec"] = map[string]interface{}{ + spec := map[string]interface{}{ "selectors": []interface{}{ map[string]interface{}{ "cel": map[string]interface{}{ - "expression": "device.driver == '" + deviceClassName + "'", + "expression": selectorExpr, }, }, }, } + if extendedResourceName != "" { + spec["extendedResourceName"] = extendedResourceName + } + dc.Object["spec"] = spec if err := dcrh.client.Create(ctx, dc); err != nil { if k8serrors.IsAlreadyExists(err) { return nil } - return fmt.Errorf("failed to create DeviceClass %s: %v", deviceClassName, err) + return fmt.Errorf("failed to create DeviceClass %s: %v", name, err) } - logger.Info("Created DeviceClass", "name", deviceClassName, "apiVersion", apiVersion) + log.FromContext(ctx).Info("Created DeviceClass", "name", name, "apiVersion", apiVersion) return nil } diff --git a/internal/controllers/device_config_reconciler_test.go b/internal/controllers/device_config_reconciler_test.go index 846b1418d..cfca72f87 100644 --- a/internal/controllers/device_config_reconciler_test.go +++ b/internal/controllers/device_config_reconciler_test.go @@ -55,6 +55,7 @@ import ( v1 "k8s.io/api/core/v1" k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" @@ -898,4 +899,62 @@ var _ = Describe("handleDeviceClass", func() { err := dcrh.handleDeviceClass(ctx, draEnabledConfig) Expect(err).To(HaveOccurred()) }) + + It("sets extendedResourceName on the single DeviceClass when AutoPartition is disabled", func() { + ctrl := gomock.NewController(GinkgoT()) + kubeClient = mock_client.NewMockClient(ctrl) + dcrh = newDeviceConfigReconcilerHelper(kubeClient, nil, nil, nil, nil, nil, nil, nil, nil, nil, true, true) + + var created *unstructured.Unstructured + kubeClient.EXPECT().Create(ctx, gomock.Any()).DoAndReturn( + func(_ context.Context, obj client.Object, _ ...client.CreateOption) error { + created = obj.(*unstructured.Unstructured) + return nil + }, + ) + + err := dcrh.handleDeviceClass(ctx, draEnabledConfig) + Expect(err).ToNot(HaveOccurred()) + Expect(created.GetName()).To(Equal("gpu.amd.com")) + ern, found, _ := unstructured.NestedString(created.Object, "spec", "extendedResourceName") + Expect(found).To(BeTrue()) + Expect(ern).To(Equal("amd.com/gpu")) + }) + + It("creates both DeviceClasses when AutoPartition is enabled, extendedResourceName only on the spx class", func() { + ctrl := gomock.NewController(GinkgoT()) + kubeClient = mock_client.NewMockClient(ctrl) + dcrh = newDeviceConfigReconcilerHelper(kubeClient, nil, nil, nil, nil, nil, nil, nil, nil, nil, true, true) + + autoPartitionConfig := &amdv1alpha1.DeviceConfig{ + ObjectMeta: metav1.ObjectMeta{Name: devConfigName, Namespace: devConfigNamespace}, + Spec: amdv1alpha1.DeviceConfigSpec{ + DRADriver: amdv1alpha1.DRADriverSpec{ + Enable: &draEnabled, + CmdLineArguments: map[string]string{"feature-gates": "AutoPartition=true"}, + }, + }, + } + + var created []*unstructured.Unstructured + kubeClient.EXPECT().Create(ctx, gomock.Any()).Times(2).DoAndReturn( + func(_ context.Context, obj client.Object, _ ...client.CreateOption) error { + created = append(created, obj.(*unstructured.Unstructured)) + return nil + }, + ) + + err := dcrh.handleDeviceClass(ctx, autoPartitionConfig) + Expect(err).ToNot(HaveOccurred()) + Expect(created).To(HaveLen(2)) + + Expect(created[0].GetName()).To(Equal("gpu.amd.com")) + _, found, _ := unstructured.NestedString(created[0].Object, "spec", "extendedResourceName") + Expect(found).To(BeFalse()) + + Expect(created[1].GetName()).To(Equal("gpu.amd.com-spx")) + ern, found, _ := unstructured.NestedString(created[1].Object, "spec", "extendedResourceName") + Expect(found).To(BeTrue()) + Expect(ern).To(Equal("amd.com/gpu")) + }) }) diff --git a/internal/controllers/upgrademgr.go b/internal/controllers/upgrademgr.go index 6a9dfae22..2e85c8645 100644 --- a/internal/controllers/upgrademgr.go +++ b/internal/controllers/upgrademgr.go @@ -1321,8 +1321,8 @@ func (h *upgradeMgrHelper) getRebootPod(nodeName string, dc *amdv1alpha1.DeviceC ImagePullSecrets: imagePullSecrets, Containers: []v1.Container{ { - Name: "reboot-container", - Image: utilsImage, + Name: "reboot-container", + Image: utilsImage, // Flush filesystems before reboot to avoid exec-format errors from unflushed overlay writes. Command: []string{"/nsenter", "--all", "--target=1", "--", "sh", "-c", "sync; sudo reboot"}, Stdin: true, diff --git a/internal/plugin/plugin.go b/internal/plugin/plugin.go index 966bd09f8..73dbdddb1 100644 --- a/internal/plugin/plugin.go +++ b/internal/plugin/plugin.go @@ -408,6 +408,14 @@ func (dp *devicePlugin) SetDRADriverAsDesired(ds *appsv1.DaemonSet, devConfig *a Name: "sys", MountPath: "/sys", }, + // Required by the AutoPartition feature's driver-reload paths + // (both non-KMM modprobe and KMM-managed) so the amdgpu kernel + // module can be resolved/reloaded from inside the container. + { + Name: "lib-modules", + MountPath: "/lib/modules", + ReadOnly: true, + }, }, }, }, @@ -456,10 +464,25 @@ func (dp *devicePlugin) SetDRADriverAsDesired(ds *appsv1.DaemonSet, devConfig *a }, }, }, + { + Name: "lib-modules", + VolumeSource: v1.VolumeSource{ + HostPath: &v1.HostPathVolumeSource{ + Path: "/lib/modules", + Type: ptr.To(v1.HostPathDirectory), + }, + }, + }, }, }, }, } + if utils.ShouldUseKMM(devConfig) { + ds.Spec.Template.Spec.Containers[0].Env = append(ds.Spec.Template.Spec.Containers[0].Env, v1.EnvVar{ + Name: "KMM_DRIVER_ENABLED", + Value: "true", + }) + } if devConfig.Spec.DRADriver.UpgradePolicy != nil { up := devConfig.Spec.DRADriver.UpgradePolicy upgradeStrategy := appsv1.RollingUpdateDaemonSetStrategyType diff --git a/internal/plugin/plugin_test.go b/internal/plugin/plugin_test.go index cb0c3f2df..4d6fd30dd 100644 --- a/internal/plugin/plugin_test.go +++ b/internal/plugin/plugin_test.go @@ -176,3 +176,118 @@ var _ = Describe("SetDevicePluginAsDesired", func() { Expect(err.Error()).To(ContainSubstring("daemon set is not initialized")) }) }) + +var _ = Describe("SetDRADriverAsDesired", func() { + var dp *devicePlugin + + BeforeEach(func() { + dp = &devicePlugin{ + client: nil, + scheme: scheme, + isOpenShift: false, + } + }) + + newDevConfig := func(driverEnable bool, driverType string) *amdv1alpha1.DeviceConfig { + return &amdv1alpha1.DeviceConfig{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-config", + Namespace: "test-namespace", + }, + Spec: amdv1alpha1.DeviceConfigSpec{ + Driver: amdv1alpha1.DriverSpec{ + Enable: &driverEnable, + DriverType: driverType, + }, + DRADriver: amdv1alpha1.DRADriverSpec{}, + }, + } + } + + // Both auto-partition reload paths (non-KMM modprobe and KMM-managed) shell + // out to modprobe inside the container, which needs /lib/modules regardless + // of whether KMM manages the driver. + It("always mounts /lib/modules for the amdgpu module reload paths", func() { + ds := &appsv1.DaemonSet{ + ObjectMeta: metav1.ObjectMeta{Name: "test-dra-driver", Namespace: "test-namespace"}, + } + devConfig := newDevConfig(false, "") + + err := dp.SetDRADriverAsDesired(ds, devConfig) + Expect(err).To(BeNil()) + + var foundVolumeMount bool + for _, vm := range ds.Spec.Template.Spec.Containers[0].VolumeMounts { + if vm.Name == "lib-modules" { + Expect(vm.MountPath).To(Equal("/lib/modules")) + Expect(vm.ReadOnly).To(BeTrue()) + foundVolumeMount = true + } + } + Expect(foundVolumeMount).To(BeTrue(), "lib-modules volume mount not found") + + var foundVolume bool + for _, vol := range ds.Spec.Template.Spec.Volumes { + if vol.Name == "lib-modules" { + Expect(vol.HostPath.Path).To(Equal("/lib/modules")) + foundVolume = true + } + } + Expect(foundVolume).To(BeTrue(), "lib-modules volume not found") + }) + + It("sets KMM_DRIVER_ENABLED when the driver is KMM-managed", func() { + ds := &appsv1.DaemonSet{ + ObjectMeta: metav1.ObjectMeta{Name: "test-dra-driver", Namespace: "test-namespace"}, + } + devConfig := newDevConfig(true, utils.DriverTypeContainer) + + err := dp.SetDRADriverAsDesired(ds, devConfig) + Expect(err).To(BeNil()) + + var found bool + for _, e := range ds.Spec.Template.Spec.Containers[0].Env { + if e.Name == "KMM_DRIVER_ENABLED" { + Expect(e.Value).To(Equal("true")) + found = true + } + } + Expect(found).To(BeTrue(), "KMM_DRIVER_ENABLED env var not set") + }) + + It("does not set KMM_DRIVER_ENABLED when the driver is not KMM-managed", func() { + ds := &appsv1.DaemonSet{ + ObjectMeta: metav1.ObjectMeta{Name: "test-dra-driver", Namespace: "test-namespace"}, + } + devConfig := newDevConfig(false, "") + + err := dp.SetDRADriverAsDesired(ds, devConfig) + Expect(err).To(BeNil()) + + for _, e := range ds.Spec.Template.Spec.Containers[0].Env { + Expect(e.Name).NotTo(Equal("KMM_DRIVER_ENABLED")) + } + }) + + It("does not set KMM_DRIVER_ENABLED for pf-passthrough driver type", func() { + ds := &appsv1.DaemonSet{ + ObjectMeta: metav1.ObjectMeta{Name: "test-dra-driver", Namespace: "test-namespace"}, + } + devConfig := newDevConfig(true, utils.DriverTypePFPassthrough) + + err := dp.SetDRADriverAsDesired(ds, devConfig) + Expect(err).To(BeNil()) + + for _, e := range ds.Spec.Template.Spec.Containers[0].Env { + Expect(e.Name).NotTo(Equal("KMM_DRIVER_ENABLED")) + } + }) + + It("should return error when daemonset is nil", func() { + devConfig := newDevConfig(false, "") + + err := dp.SetDRADriverAsDesired(nil, devConfig) + Expect(err).NotTo(BeNil()) + Expect(err.Error()).To(ContainSubstring("daemon set is not initialized")) + }) +}) diff --git a/internal/utils.go b/internal/utils.go index bf66674f3..11374b6bd 100644 --- a/internal/utils.go +++ b/internal/utils.go @@ -506,3 +506,23 @@ func ShouldUseKMM(devConfig *v1alpha1.DeviceConfig) bool { } return false } + +// ShouldUseAutoPartition reports whether the DRA driver's AutoPartition feature +// gate is enabled via DeviceConfig.spec.draDriver.cmdLineArguments, e.g. +// cmdLineArguments: {feature-gates: "AutoPartition=true"}. This is the only +// place AutoPartition is configured today -- there is no dedicated DeviceConfig +// field for it -- so this parses the same "key=value,key=value" string the DRA +// driver's own --feature-gates flag accepts. +func ShouldUseAutoPartition(devConfig *v1alpha1.DeviceConfig) bool { + if devConfig == nil { + return false + } + gates := devConfig.Spec.DRADriver.CmdLineArguments["feature-gates"] + for _, pair := range strings.Split(gates, ",") { + kv := strings.SplitN(strings.TrimSpace(pair), "=", 2) + if len(kv) == 2 && kv[0] == "AutoPartition" { + return kv[1] == "true" + } + } + return false +}