From 9aceb842760716cd00b6a6685b655218f165fb34 Mon Sep 17 00:00:00 2001 From: devivasudevan <49675305+devivasudevan@users.noreply.github.com> Date: Wed, 7 Oct 2026 13:54:33 +0200 Subject: [PATCH 01/40] feat(operator): add dgdr-run-publisher sidecar and DGDC reconcile Reads the pod-local dgdr_run_snapshot.yaml written by the dgdr_run output adapter and publishes it on the DGDR run: creates immutable DGDCs with stable, rank-independent ids, patches run status (message, progress, ordered candidate refs), then deletes candidates that left the top N, in that order. The terminal snapshot is the completion barrier; sweeper exit codes and a missing or invalid terminal snapshot map to publisher exit codes 0/1/3. The v1beta2 types are placeholders under internal/dgdrrunpublisher/placeholderapi until the DGDR v1beta2 API lands. --- .../operator/cmd/dgdr-run-publisher/main.go | 98 +++ .../operator/internal/dgdcreconcile/diff.go | 108 ++++ .../internal/dgdcreconcile/diff_test.go | 114 ++++ .../internal/dgdrrunpublisher/kube.go | 187 ++++++ .../dgdrrunpublisher/placeholderapi/types.go | 275 +++++++++ .../internal/dgdrrunpublisher/publisher.go | 319 ++++++++++ .../dgdrrunpublisher/publisher_test.go | 559 ++++++++++++++++++ .../internal/dgdrrunpublisher/snapshot.go | 118 ++++ 8 files changed, 1778 insertions(+) create mode 100644 deploy/operator/cmd/dgdr-run-publisher/main.go create mode 100644 deploy/operator/internal/dgdcreconcile/diff.go create mode 100644 deploy/operator/internal/dgdcreconcile/diff_test.go create mode 100644 deploy/operator/internal/dgdrrunpublisher/kube.go create mode 100644 deploy/operator/internal/dgdrrunpublisher/placeholderapi/types.go create mode 100644 deploy/operator/internal/dgdrrunpublisher/publisher.go create mode 100644 deploy/operator/internal/dgdrrunpublisher/publisher_test.go create mode 100644 deploy/operator/internal/dgdrrunpublisher/snapshot.go diff --git a/deploy/operator/cmd/dgdr-run-publisher/main.go b/deploy/operator/cmd/dgdr-run-publisher/main.go new file mode 100644 index 000000000000..8140905a4dd0 --- /dev/null +++ b/deploy/operator/cmd/dgdr-run-publisher/main.go @@ -0,0 +1,98 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Command dgdr-run-publisher is the sidecar of a DGDR(v2) run Job. It watches the +// Sweeper's pod-local snapshot file and reconciles it into DynamoGraphDeploymentCandidates +// and DynamoGraphDeploymentRun.status. See internal/dgdrrunpublisher. +package main + +import ( + "context" + "flag" + "fmt" + "log" + "os" + "os/signal" + "syscall" + "time" + + v1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/dgdrrunpublisher" + v1beta2 "github.com/ai-dynamo/dynamo/deploy/operator/internal/dgdrrunpublisher/placeholderapi" + "k8s.io/apimachinery/pkg/runtime" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +func main() { + os.Exit(run()) +} + +func run() int { + var ( + snapshotDir string + runName string + sweeperContainer string + pollInterval time.Duration + ) + flag.StringVar(&snapshotDir, "snapshot-dir", "", "directory shared with the Sweeper container") + flag.StringVar(&runName, "run-name", "", "name of the DynamoGraphDeploymentRun") + flag.StringVar(&sweeperContainer, "sweeper-container", "sweeper", "name of the Sweeper container in this pod") + flag.DurationVar(&pollInterval, "poll-interval", time.Second, "snapshot poll interval") + flag.Parse() + + namespace, podName := os.Getenv("POD_NAMESPACE"), os.Getenv("POD_NAME") + if snapshotDir == "" || runName == "" || namespace == "" || podName == "" { + fmt.Fprintln(os.Stderr, "--snapshot-dir, --run-name and the POD_NAMESPACE/POD_NAME environment variables are required") + return dgdrrunpublisher.ExitReconcileFailed + } + + scheme := runtime.NewScheme() + for _, add := range []func(*runtime.Scheme) error{clientgoscheme.AddToScheme, v1beta1.AddToScheme, v1beta2.AddToScheme} { + if err := add(scheme); err != nil { + log.Printf("building scheme: %v", err) + return dgdrrunpublisher.ExitReconcileFailed + } + } + kube, err := client.New(ctrl.GetConfigOrDie(), client.Options{Scheme: scheme}) + if err != nil { + log.Printf("creating client: %v", err) + return dgdrrunpublisher.ExitReconcileFailed + } + + ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGTERM, syscall.SIGINT) + defer stop() + + publisher := &dgdrrunpublisher.Publisher{ + Cluster: &dgdrrunpublisher.KubeCluster{ + Client: kube, + Namespace: namespace, + RunName: runName, + PodName: podName, + SweeperContainer: sweeperContainer, + }, + SnapshotDir: snapshotDir, + RunName: runName, + PollInterval: pollInterval, + } + err = publisher.Run(ctx) + if err != nil { + log.Printf("publisher: %v", err) + } + return dgdrrunpublisher.ExitCode(err) +} diff --git a/deploy/operator/internal/dgdcreconcile/diff.go b/deploy/operator/internal/dgdcreconcile/diff.go new file mode 100644 index 000000000000..6120b8a4ecc4 --- /dev/null +++ b/deploy/operator/internal/dgdcreconcile/diff.go @@ -0,0 +1,108 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package dgdcreconcile computes the create/delete diff between the candidates a run +// snapshot wants to exist and the DynamoGraphDeploymentCandidates (DGDCs) observed in +// the cluster. +// +// Identity is the stable candidate id carried by the snapshot, which depends only on +// the evaluated point -- never on rank or list position. A DGDC is immutable once its +// initial status is populated, so an identity present on both sides needs no action: +// there is deliberately no "update" action. Reordering candidates is expressed only +// through the ordered candidate references on the run status, which this package does +// not touch. +// +// The package is pure (no Kubernetes types, no I/O) so the diff can be tested on its +// own; callers fetch current state and apply the returned Actions. +package dgdcreconcile + +import "fmt" + +// DiffInputError means the desired or current set was malformed (e.g. a duplicate or +// empty identity). Retrying will not fix it, so callers treat it as terminal. +type DiffInputError struct { + msg string +} + +func (e *DiffInputError) Error() string { return e.msg } + +func newDiffInputError(format string, args ...any) *DiffInputError { + return &DiffInputError{msg: fmt.Sprintf(format, args...)} +} + +// DesiredCandidate is one candidate the snapshot wants to exist. +type DesiredCandidate struct { + // ID is the stable candidate identity from the snapshot. + ID string + // Spec is the DGD manifest (YAML) to materialize. + Spec string + // Parameters and Metrics are the immutable evaluation facts recorded in the + // DGDC status at creation. + Parameters map[string]any + Metrics map[string]any +} + +// CurrentDGDC is one DGDC observed in the cluster. ID is read back from the label +// recorded at creation, never recomputed. +type CurrentDGDC struct { + Name string + ID string +} + +// Actions is what the caller must apply. There are no updates: DGDCs are immutable. +type Actions struct { + Creates []DesiredCandidate + Deletes []string // DGDC names +} + +// ComputeActions diffs desired against current. Output order follows the input order +// so results are deterministic. +func ComputeActions(desired []DesiredCandidate, current []CurrentDGDC) (Actions, error) { + desiredIDs := make(map[string]struct{}, len(desired)) + for _, candidate := range desired { + if candidate.ID == "" { + return Actions{}, newDiffInputError("desired candidate has an empty id") + } + if _, dup := desiredIDs[candidate.ID]; dup { + return Actions{}, newDiffInputError("duplicate id in desired set: %s", candidate.ID) + } + desiredIDs[candidate.ID] = struct{}{} + } + currentIDs := make(map[string]struct{}, len(current)) + for _, existing := range current { + if existing.ID == "" { + return Actions{}, newDiffInputError("DGDC %s has no candidate id", existing.Name) + } + if _, dup := currentIDs[existing.ID]; dup { + return Actions{}, newDiffInputError("duplicate id in current set: %s", existing.ID) + } + currentIDs[existing.ID] = struct{}{} + } + + var actions Actions + for _, candidate := range desired { + if _, exists := currentIDs[candidate.ID]; !exists { + actions.Creates = append(actions.Creates, candidate) + } + } + for _, existing := range current { + if _, wanted := desiredIDs[existing.ID]; !wanted { + actions.Deletes = append(actions.Deletes, existing.Name) + } + } + return actions, nil +} diff --git a/deploy/operator/internal/dgdcreconcile/diff_test.go b/deploy/operator/internal/dgdcreconcile/diff_test.go new file mode 100644 index 000000000000..6a5358b0a399 --- /dev/null +++ b/deploy/operator/internal/dgdcreconcile/diff_test.go @@ -0,0 +1,114 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dgdcreconcile + +import ( + "errors" + "reflect" + "testing" +) + +func d(ids ...string) []DesiredCandidate { + out := make([]DesiredCandidate, 0, len(ids)) + for _, id := range ids { + out = append(out, DesiredCandidate{ID: id, Spec: "spec-" + id}) + } + return out +} + +func c(pairs ...string) []CurrentDGDC { + var out []CurrentDGDC + for i := 0; i+1 < len(pairs); i += 2 { + out = append(out, CurrentDGDC{Name: pairs[i], ID: pairs[i+1]}) + } + return out +} + +func createIDs(a Actions) []string { + var ids []string + for _, x := range a.Creates { + ids = append(ids, x.ID) + } + return ids +} + +func TestComputeActions(t *testing.T) { + tests := []struct { + name string + desired []DesiredCandidate + current []CurrentDGDC + wantCreates []string + wantDeletes []string + }{ + {"empty both", nil, nil, nil, nil}, + {"all new", d("a", "b"), nil, []string{"a", "b"}, nil}, + {"all gone", nil, c("n-a", "a", "n-b", "b"), nil, []string{"n-a", "n-b"}}, + {"steady state needs no action", d("a", "b"), c("n-a", "a", "n-b", "b"), nil, nil}, + {"reordering desired needs no action", d("b", "a"), c("n-a", "a", "n-b", "b"), nil, nil}, + {"mixed", d("a", "c"), c("n-a", "a", "n-b", "b"), []string{"c"}, []string{"n-b"}}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + got, err := ComputeActions(tc.desired, tc.current) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !reflect.DeepEqual(createIDs(got), tc.wantCreates) { + t.Errorf("creates = %v, want %v", createIDs(got), tc.wantCreates) + } + if !reflect.DeepEqual(got.Deletes, tc.wantDeletes) { + t.Errorf("deletes = %v, want %v", got.Deletes, tc.wantDeletes) + } + }) + } +} + +func TestComputeActionsInputErrors(t *testing.T) { + tests := []struct { + name string + desired []DesiredCandidate + current []CurrentDGDC + }{ + {"duplicate desired", d("a", "a"), nil}, + {"empty desired id", d(""), nil}, + {"duplicate current", nil, c("n1", "a", "n2", "a")}, + {"empty current id", nil, c("n1", "")}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + _, err := ComputeActions(tc.desired, tc.current) + var inputErr *DiffInputError + if !errors.As(err, &inputErr) { + t.Fatalf("want *DiffInputError, got %v", err) + } + }) + } +} + +func TestComputeActionsDeterministicOrder(t *testing.T) { + got, err := ComputeActions(d("z", "m", "a"), c("n-q", "q", "n-b", "b")) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(createIDs(got), []string{"z", "m", "a"}) { + t.Errorf("creates order = %v", createIDs(got)) + } + if !reflect.DeepEqual(got.Deletes, []string{"n-q", "n-b"}) { + t.Errorf("deletes order = %v", got.Deletes) + } +} diff --git a/deploy/operator/internal/dgdrrunpublisher/kube.go b/deploy/operator/internal/dgdrrunpublisher/kube.go new file mode 100644 index 000000000000..4f3d3913f2e3 --- /dev/null +++ b/deploy/operator/internal/dgdrrunpublisher/kube.go @@ -0,0 +1,187 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dgdrrunpublisher + +import ( + "context" + "encoding/json" + "fmt" + + v1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/dgdcreconcile" + v1beta2 "github.com/ai-dynamo/dynamo/deploy/operator/internal/dgdrrunpublisher/placeholderapi" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/yaml" +) + +const ( + // LabelRunName ties a candidate to its run. + LabelRunName = "nvidia.com/dgdr-run-name" + // LabelCandidateID records the stable candidate id (the identity read back on list). + LabelCandidateID = "nvidia.com/dgdr-candidate-id" +) + +// KubeCluster is the Cluster implementation backed by a controller-runtime client. It +// should be a direct (uncached) client: the publisher is a short-lived sidecar, so it +// neither needs nor wants an informer cache. +// +// Required RBAC (namespaced, Role bound to the run's ServiceAccount): candidates +// get/list/create/delete and candidates/status update; runs get and runs/status patch; +// pods get (own pod, for Sweeper crash detection; resourceNames cannot be used because +// Job pod names are generated). +type KubeCluster struct { + Client client.Client + Namespace string + RunName string + PodName string + SweeperContainer string +} + +var _ Cluster = (*KubeCluster)(nil) + +func (k *KubeCluster) ListCandidates(ctx context.Context) ([]dgdcreconcile.CurrentDGDC, error) { + var list v1beta2.DynamoGraphDeploymentCandidateList + if err := k.Client.List(ctx, &list, client.InNamespace(k.Namespace), client.MatchingLabels{LabelRunName: k.RunName}); err != nil { + return nil, err + } + out := make([]dgdcreconcile.CurrentDGDC, 0, len(list.Items)) + for i := range list.Items { + out = append(out, dgdcreconcile.CurrentDGDC{ + Name: list.Items[i].Name, + ID: list.Items[i].Labels[LabelCandidateID], + }) + } + return out, nil +} + +// CreateCandidate creates the candidate and populates its status exactly once. Both +// steps are idempotent so a retry after a partial failure converges. +func (k *KubeCluster) CreateCandidate(ctx context.Context, name string, candidate dgdcreconcile.DesiredCandidate) error { + var doc struct { + Spec v1beta1.DynamoGraphDeploymentSpec `json:"spec"` + } + if err := yaml.Unmarshal([]byte(candidate.Spec), &doc); err != nil { + return fmt.Errorf("decoding manifest of candidate %s: %w", candidate.ID, err) + } + var run v1beta2.DynamoGraphDeploymentRun + if err := k.Client.Get(ctx, types.NamespacedName{Namespace: k.Namespace, Name: k.RunName}, &run); err != nil { + return fmt.Errorf("getting run: %w", err) + } + parameters, err := rawExtension(candidate.Parameters) + if err != nil { + return err + } + object := &v1beta2.DynamoGraphDeploymentCandidate{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: k.Namespace, + Name: name, + Labels: map[string]string{LabelRunName: k.RunName, LabelCandidateID: candidate.ID}, + }, + Spec: v1beta2.DynamoGraphDeploymentCandidateSpec{ + DynamoGraphDeploymentSpec: doc.Spec, + Parameters: parameters, + }, + } + if err := controllerutil.SetControllerReference(&run, object, k.Client.Scheme()); err != nil { + return err + } + if err := k.Client.Create(ctx, object); err != nil && !apierrors.IsAlreadyExists(err) { + return err + } + + var stored v1beta2.DynamoGraphDeploymentCandidate + if err := k.Client.Get(ctx, types.NamespacedName{Namespace: k.Namespace, Name: name}, &stored); err != nil { + return err + } + if len(stored.Status.Conditions) > 0 { + return nil // status already populated; candidates are immutable afterwards + } + metrics, err := rawExtension(candidate.Metrics) + if err != nil { + return err + } + stored.Status.Metrics = metrics + meta.SetStatusCondition(&stored.Status.Conditions, metav1.Condition{ + Type: "Evaluated", + Status: metav1.ConditionTrue, + Reason: "Evaluated", + Message: "evaluated by the Sweeper", + }) + return k.Client.Status().Update(ctx, &stored) +} + +func rawExtension(value map[string]any) (*runtime.RawExtension, error) { + if value == nil { + value = map[string]any{} + } + raw, err := json.Marshal(value) + if err != nil { + return nil, err + } + return &runtime.RawExtension{Raw: raw}, nil +} + +func (k *KubeCluster) DeleteCandidate(ctx context.Context, name string) error { + object := &v1beta2.DynamoGraphDeploymentCandidate{ObjectMeta: metav1.ObjectMeta{Namespace: k.Namespace, Name: name}} + return client.IgnoreNotFound(k.Client.Delete(ctx, object)) +} + +func (k *KubeCluster) PatchRunStatus(ctx context.Context, status RunStatus) error { + var run v1beta2.DynamoGraphDeploymentRun + if err := k.Client.Get(ctx, types.NamespacedName{Namespace: k.Namespace, Name: k.RunName}, &run); err != nil { + return err + } + base := run.DeepCopy() + run.Status.Message = status.Message + run.Status.Progress = &v1beta2.RunProgress{Round: status.Round, Evaluated: status.Evaluated} + if !status.LastProgress.IsZero() { + progressed := metav1.NewTime(status.LastProgress) + run.Status.LastProgressTime = &progressed + } + refs := make([]v1beta2.CandidateRef, 0, len(status.CandidateNames)) + for _, name := range status.CandidateNames { + refs = append(refs, v1beta2.CandidateRef{Name: name}) + } + run.Status.CandidateRefs = refs + return k.Client.Status().Patch(ctx, &run, client.MergeFrom(base)) +} + +// SweeperState reads the Sweeper container's status from this pod's own status. +func (k *KubeCluster) SweeperState(ctx context.Context) (SweeperState, error) { + var pod corev1.Pod + if err := k.Client.Get(ctx, types.NamespacedName{Namespace: k.Namespace, Name: k.PodName}, &pod); err != nil { + return SweeperState{}, err + } + for _, status := range pod.Status.ContainerStatuses { + if status.Name != k.SweeperContainer { + continue + } + if terminated := status.State.Terminated; terminated != nil { + return SweeperState{Exited: true, ExitCode: terminated.ExitCode}, nil + } + return SweeperState{}, nil + } + return SweeperState{}, nil +} diff --git a/deploy/operator/internal/dgdrrunpublisher/placeholderapi/types.go b/deploy/operator/internal/dgdrrunpublisher/placeholderapi/types.go new file mode 100644 index 000000000000..8541668797d7 --- /dev/null +++ b/deploy/operator/internal/dgdrrunpublisher/placeholderapi/types.go @@ -0,0 +1,275 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// PLACEHOLDER -- not the real v1beta2 API (and deliberately not under api/, so it is not a served +// or converted API version). This package exists only so the DGDR(v2) +// run publisher (internal/dgdrrunpublisher) has something to compile against before the +// real v1beta2 types exist on this branch. The real, reviewed types are PRs #13603 +// ("feat: add api for dgdr v1beta2") and #13744 ("feat: dgdr v1beta2 conversion"), +// still feature-gated/inactive when this file was written. The shapes below follow the +// review of #15554: a candidate is an immutable evaluated point (spec = the DGD fields +// plus the resolved evaluation parameters, status = metrics + evaluation conditions) +// and ordering lives on the run's status, never on the candidate. Delete this package when the real types land; +// only import paths change. +// +// Deliberately NOT included: conversion webhooks, the full Request type, and +// deepcopy-gen output (DeepCopyObject is hand-written here). +package placeholderapi + +import ( + v1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" +) + +// GroupVersion is the group version used to register these placeholder objects. +var GroupVersion = schema.GroupVersion{Group: "nvidia.com", Version: "v1beta2"} + +// AddToScheme registers the placeholder types with the given scheme. +func AddToScheme(scheme *runtime.Scheme) error { + scheme.AddKnownTypes(GroupVersion, + &DynamoGraphDeploymentCandidate{}, + &DynamoGraphDeploymentCandidateList{}, + &DynamoGraphDeploymentRun{}, + &DynamoGraphDeploymentRunList{}, + ) + metav1.AddToGroupVersion(scheme, GroupVersion) + return nil +} + +// DynamoGraphDeploymentCandidateSpec is the immutable evaluated point: the flat DGD +// fields plus the resolved evaluation parameters. +type DynamoGraphDeploymentCandidateSpec struct { + v1beta1.DynamoGraphDeploymentSpec `json:",inline"` + + // Parameters are the resolved evaluation parameters of the point. + // +optional + Parameters *runtime.RawExtension `json:"parameters,omitempty"` +} + +// DynamoGraphDeploymentCandidateStatus records evaluation results. It is populated once +// after creation and never updated afterwards; it carries no rank. +type DynamoGraphDeploymentCandidateStatus struct { + // Metrics are the evaluation metrics of the point. + // +optional + Metrics *runtime.RawExtension `json:"metrics,omitempty"` + + // Conditions describes evaluation and materialization. + // +optional + Conditions []metav1.Condition `json:"conditions,omitempty"` +} + +// DynamoGraphDeploymentCandidate is one bounded, user-visible search result. +type DynamoGraphDeploymentCandidate struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec DynamoGraphDeploymentCandidateSpec `json:"spec"` + Status DynamoGraphDeploymentCandidateStatus `json:"status,omitempty"` +} + +// DynamoGraphDeploymentCandidateList contains a list of candidates. +type DynamoGraphDeploymentCandidateList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []DynamoGraphDeploymentCandidate `json:"items"` +} + +// CandidateRef is an ordered reference from a run to one of its candidates. +type CandidateRef struct { + Name string `json:"name"` +} + +// RunProgress is the Sweeper progress reported by the snapshot. +type RunProgress struct { + Round int32 `json:"round"` + Evaluated int32 `json:"evaluated"` +} + +// DynamoGraphDeploymentRunSpec is intentionally empty in this placeholder: the real +// spec is an alias of the request spec (see #13603). +type DynamoGraphDeploymentRunSpec struct{} + +// DynamoGraphDeploymentRunStatus holds everything that changes while a search runs. +// Rank is the order of CandidateRefs (best first for scalar searches). The terminal +// Completed condition is derived by the run controller from the Job result, not +// written by the publisher. +type DynamoGraphDeploymentRunStatus struct { + // Message is a human-readable summary of the latest state or failure. + // +optional + Message string `json:"message,omitempty"` + + // Progress is the latest Sweeper progress. + // +optional + Progress *RunProgress `json:"progress,omitempty"` + + // LastProgressTime is when the Sweeper last published new state. + // +optional + LastProgressTime *metav1.Time `json:"lastProgressTime,omitempty"` + + // CandidateRefs lists the run's candidates in rank order. + // +optional + CandidateRefs []CandidateRef `json:"candidateRefs,omitempty"` + + // Conditions describes the run. + // +optional + Conditions []metav1.Condition `json:"conditions,omitempty"` +} + +// DynamoGraphDeploymentRun is one execution of a search request. +type DynamoGraphDeploymentRun struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec DynamoGraphDeploymentRunSpec `json:"spec,omitempty"` + Status DynamoGraphDeploymentRunStatus `json:"status,omitempty"` +} + +// DynamoGraphDeploymentRunList contains a list of runs. +type DynamoGraphDeploymentRunList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []DynamoGraphDeploymentRun `json:"items"` +} + +// -- hand-written runtime.Object plumbing (deepcopy-gen output in the real PR) -- + +func (in *DynamoGraphDeploymentCandidateSpec) DeepCopyInto(out *DynamoGraphDeploymentCandidateSpec) { + *out = *in + in.DynamoGraphDeploymentSpec.DeepCopyInto(&out.DynamoGraphDeploymentSpec) + if in.Parameters != nil { + out.Parameters = in.Parameters.DeepCopy() + } +} + +func (in *DynamoGraphDeploymentCandidateStatus) DeepCopyInto(out *DynamoGraphDeploymentCandidateStatus) { + *out = *in + if in.Metrics != nil { + out.Metrics = in.Metrics.DeepCopy() + } + if in.Conditions != nil { + out.Conditions = make([]metav1.Condition, len(in.Conditions)) + for i := range in.Conditions { + in.Conditions[i].DeepCopyInto(&out.Conditions[i]) + } + } +} + +func (in *DynamoGraphDeploymentCandidate) DeepCopyInto(out *DynamoGraphDeploymentCandidate) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +func (in *DynamoGraphDeploymentCandidate) DeepCopy() *DynamoGraphDeploymentCandidate { + if in == nil { + return nil + } + out := new(DynamoGraphDeploymentCandidate) + in.DeepCopyInto(out) + return out +} + +func (in *DynamoGraphDeploymentCandidate) DeepCopyObject() runtime.Object { return in.DeepCopy() } + +func (in *DynamoGraphDeploymentCandidateList) DeepCopyInto(out *DynamoGraphDeploymentCandidateList) { + *out = *in + out.TypeMeta = in.TypeMeta + out.ListMeta = in.ListMeta + if in.Items != nil { + out.Items = make([]DynamoGraphDeploymentCandidate, len(in.Items)) + for i := range in.Items { + in.Items[i].DeepCopyInto(&out.Items[i]) + } + } +} + +func (in *DynamoGraphDeploymentCandidateList) DeepCopy() *DynamoGraphDeploymentCandidateList { + if in == nil { + return nil + } + out := new(DynamoGraphDeploymentCandidateList) + in.DeepCopyInto(out) + return out +} + +func (in *DynamoGraphDeploymentCandidateList) DeepCopyObject() runtime.Object { return in.DeepCopy() } + +func (in *DynamoGraphDeploymentRunStatus) DeepCopyInto(out *DynamoGraphDeploymentRunStatus) { + *out = *in + if in.Progress != nil { + progress := *in.Progress + out.Progress = &progress + } + if in.LastProgressTime != nil { + out.LastProgressTime = in.LastProgressTime.DeepCopy() + } + if in.CandidateRefs != nil { + out.CandidateRefs = append([]CandidateRef(nil), in.CandidateRefs...) + } + if in.Conditions != nil { + out.Conditions = make([]metav1.Condition, len(in.Conditions)) + for i := range in.Conditions { + in.Conditions[i].DeepCopyInto(&out.Conditions[i]) + } + } +} + +func (in *DynamoGraphDeploymentRun) DeepCopyInto(out *DynamoGraphDeploymentRun) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + out.Spec = in.Spec + in.Status.DeepCopyInto(&out.Status) +} + +func (in *DynamoGraphDeploymentRun) DeepCopy() *DynamoGraphDeploymentRun { + if in == nil { + return nil + } + out := new(DynamoGraphDeploymentRun) + in.DeepCopyInto(out) + return out +} + +func (in *DynamoGraphDeploymentRun) DeepCopyObject() runtime.Object { return in.DeepCopy() } + +func (in *DynamoGraphDeploymentRunList) DeepCopyInto(out *DynamoGraphDeploymentRunList) { + *out = *in + out.TypeMeta = in.TypeMeta + out.ListMeta = in.ListMeta + if in.Items != nil { + out.Items = make([]DynamoGraphDeploymentRun, len(in.Items)) + for i := range in.Items { + in.Items[i].DeepCopyInto(&out.Items[i]) + } + } +} + +func (in *DynamoGraphDeploymentRunList) DeepCopy() *DynamoGraphDeploymentRunList { + if in == nil { + return nil + } + out := new(DynamoGraphDeploymentRunList) + in.DeepCopyInto(out) + return out +} + +func (in *DynamoGraphDeploymentRunList) DeepCopyObject() runtime.Object { return in.DeepCopy() } diff --git a/deploy/operator/internal/dgdrrunpublisher/publisher.go b/deploy/operator/internal/dgdrrunpublisher/publisher.go new file mode 100644 index 000000000000..b8694002a32f --- /dev/null +++ b/deploy/operator/internal/dgdrrunpublisher/publisher.go @@ -0,0 +1,319 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package dgdrrunpublisher is the sidecar that turns the Sweeper's pod-local desired +// state snapshot into Kubernetes state: it creates/deletes DynamoGraphDeploymentCandidates +// and patches the ordered candidate references and progress on +// DynamoGraphDeploymentRun.status. +// +// It runs as a regular container of the same Job pod as the Sweeper, sharing an +// emptyDir with it. There is no ConfigMap and no cross-pod channel: the file is the +// interface. Because the publisher is a regular container, the Job only completes once +// the publisher has exited, so its successful exit is the acknowledgement that the final +// snapshot was reconciled. The run controller derives the terminal Completed condition +// from the Job result; the publisher never writes a phase. +// +// Termination state machine (see Publisher.Run): +// +// terminal snapshot reconciled -> exit 0 +// Sweeper exited non-zero, no terminal snapshot -> reconcile last snapshot, exit 0 +// (the Sweeper's exit fails the Job) +// Sweeper exited zero, no terminal snapshot -> reconcile last snapshot, +// exit ExitMissingTerminalSnapshot +// final/last snapshot cannot be reconciled -> exit ExitReconcileFailed +package dgdrrunpublisher + +import ( + "context" + "errors" + "fmt" + "log" + "os" + "path/filepath" + "reflect" + "time" + + "github.com/ai-dynamo/dynamo/deploy/operator/internal/dgdcreconcile" +) + +// Process exit codes of the publisher binary. +const ( + ExitAcknowledged = 0 + ExitReconcileFailed = 1 + ExitMissingTerminalSnapshot = 3 +) + +// ErrMissingTerminalSnapshot means the Sweeper exited successfully without publishing a +// terminal snapshot, which violates the producer protocol. +var ErrMissingTerminalSnapshot = errors.New("sweeper exited without publishing a terminal snapshot") + +// ExitCode maps the result of Run to the process exit code. +func ExitCode(err error) int { + switch { + case err == nil: + return ExitAcknowledged + case errors.Is(err, ErrMissingTerminalSnapshot): + return ExitMissingTerminalSnapshot + default: + return ExitReconcileFailed + } +} + +// SnapshotFileName is the file the Sweeper container writes. +const SnapshotFileName = "dgdr_run_snapshot.yaml" + +// RunStatus is the part of DynamoGraphDeploymentRun.status the publisher owns: the +// ordered candidate references plus progress. CandidateNames is in rank order (best +// first for scalar searches). LastProgress is the zero time when the snapshot carries +// no usable timestamp, in which case the existing value is left untouched. +type RunStatus struct { + Message string + Round int32 + Evaluated int32 + LastProgress time.Time + CandidateNames []string +} + +// SweeperState is the observed state of the Sweeper container in this pod. +type SweeperState struct { + Exited bool + ExitCode int32 +} + +// Cluster is everything the publisher needs from Kubernetes. +type Cluster interface { + ListCandidates(ctx context.Context) ([]dgdcreconcile.CurrentDGDC, error) + // CreateCandidate must be idempotent: an already-existing candidate is success. + CreateCandidate(ctx context.Context, name string, candidate dgdcreconcile.DesiredCandidate) error + DeleteCandidate(ctx context.Context, name string) error + PatchRunStatus(ctx context.Context, status RunStatus) error + // SweeperState reads the Sweeper container's status from this pod. + SweeperState(ctx context.Context) (SweeperState, error) +} + +// Publisher reconciles snapshots into the cluster. +type Publisher struct { + Cluster Cluster + SnapshotDir string + RunName string + PollInterval time.Duration + + lastRaw []byte + lastStatus *RunStatus + haveSnapshot bool // a snapshot has been reconciled + terminal bool // the reconciled snapshot was terminal +} + +// CandidateName is the DGDC name for a candidate id. It depends only on the run and the +// stable id, never on rank. +func CandidateName(runName, id string) string { + return fmt.Sprintf("%s-%s", runName, id) +} + +// Reconcile applies one snapshot: create missing candidates, patch the run status +// (ordered references and progress), then delete obsolete candidates. Existing +// candidates are never updated. It is idempotent and safe to retry. +func (p *Publisher) Reconcile(ctx context.Context, snap *Snapshot) error { + var desired []dgdcreconcile.DesiredCandidate + var names []string + failed := 0 + for _, candidate := range snap.Candidates { + if candidate.Outcome != OutcomeMaterialized { + failed++ + continue + } + desired = append(desired, dgdcreconcile.DesiredCandidate{ + ID: candidate.ID, + Spec: candidate.Manifest, + Parameters: candidate.Parameters, + Metrics: candidate.Metrics, + }) + names = append(names, CandidateName(p.RunName, candidate.ID)) + } + + current, err := p.Cluster.ListCandidates(ctx) + if err != nil { + return fmt.Errorf("listing candidates: %w", err) + } + actions, err := dgdcreconcile.ComputeActions(desired, current) + if err != nil { + return err + } + for _, candidate := range actions.Creates { + if err := p.Cluster.CreateCandidate(ctx, CandidateName(p.RunName, candidate.ID), candidate); err != nil { + return fmt.Errorf("creating candidate %s: %w", candidate.ID, err) + } + } + + status := RunStatus{ + Message: runMessage(snap, failed), + Round: snap.Progress.Round, + Evaluated: snap.Progress.Evaluated, + LastProgress: progressTime(snap), + CandidateNames: names, + } + if err := p.patchStatus(ctx, status); err != nil { + return err + } + + for _, name := range actions.Deletes { + if err := p.Cluster.DeleteCandidate(ctx, name); err != nil { + return fmt.Errorf("deleting candidate %s: %w", name, err) + } + } + return nil +} + +func runMessage(snap *Snapshot, failed int) string { + message := snap.Run.Message + if snap.Run.Error != "" { + message = snap.Run.Error + } + if failed > 0 { + suffix := fmt.Sprintf("%d candidate(s) failed materialization", failed) + if message == "" { + return suffix + } + return message + "; " + suffix + } + return message +} + +// progressTime is the snapshot's own timestamp (written by the Sweeper container in the +// same pod), or the zero time when it is missing or malformed. Using the snapshot's time +// instead of the clock keeps reconciliation deterministic. +func progressTime(snap *Snapshot) time.Time { + parsed, err := time.Parse(time.RFC3339, snap.Timestamp) + if err != nil { + return time.Time{} + } + return parsed.UTC() +} + +// patchStatus skips the API call when nothing changed since the last successful patch, +// so repeated identical snapshots cause no writes. +func (p *Publisher) patchStatus(ctx context.Context, status RunStatus) error { + if p.lastStatus != nil && reflect.DeepEqual(*p.lastStatus, status) { + return nil + } + if err := p.Cluster.PatchRunStatus(ctx, status); err != nil { + return fmt.Errorf("patching run status: %w", err) + } + p.lastStatus = &status + return nil +} + +// syncOnce reads the snapshot file and reconciles it if it changed. A missing file or +// an unchanged snapshot is not an error. +func (p *Publisher) syncOnce(ctx context.Context) error { + data, err := os.ReadFile(filepath.Join(p.SnapshotDir, SnapshotFileName)) + if errors.Is(err, os.ErrNotExist) { + return nil + } + if err != nil { + return err + } + if p.lastRaw != nil && string(p.lastRaw) == string(data) { + return nil + } + snap, err := ParseSnapshot(data) + if err != nil { + return err + } + if err := p.Reconcile(ctx, snap); err != nil { + return err + } + // Record exactly the bytes that were reconciled, even if the file was replaced + // since: the next sync will see the newer content as a change. + p.lastRaw = data + p.haveSnapshot = true + p.terminal = snap.Run.Terminal + return nil +} + +// Run consumes snapshots until a terminal snapshot has been reconciled or the Sweeper +// container has exited. See the package comment for the exit semantics. +func (p *Publisher) Run(ctx context.Context) error { + ticker := time.NewTicker(p.retryDelay()) + defer ticker.Stop() + + for { + // Read the container state BEFORE the snapshot: the Sweeper's last write + // precedes its exit, so an exit observed here means that write is visible below. + state, stateErr := p.Cluster.SweeperState(ctx) + if stateErr != nil { + log.Printf("reading sweeper container state: %v", stateErr) + } + syncErr := p.syncOnce(ctx) + var inputErr *dgdcreconcile.DiffInputError + if errors.As(syncErr, &inputErr) { + return syncErr // malformed projection; retrying cannot fix it + } + if syncErr != nil { + log.Printf("sync: %v", syncErr) + } + + if syncErr == nil && p.terminal { + return nil + } + if stateErr == nil && state.Exited { + return p.finish(ctx, state, syncErr) + } + + select { + case <-ctx.Done(): + return ctx.Err() + case <-ticker.C: + } + } +} + +// finish handles the Sweeper having exited without a reconciled terminal snapshot. Any +// snapshot it ever wrote is now visible, so the last one is reconciled (with retries, +// since nothing will rewrite it) before deciding the outcome. +func (p *Publisher) finish(ctx context.Context, state SweeperState, syncErr error) error { + for attempt := 1; syncErr != nil; attempt++ { + if attempt >= finalReconcileAttempts { + return fmt.Errorf("reconciling last snapshot: %w", syncErr) + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(p.retryDelay()): + } + syncErr = p.syncOnce(ctx) + } + switch { + case p.terminal: + return nil + case state.ExitCode != 0: + // Crash or caught failure without a terminal snapshot: the last-known projection + // is reconciled above; the Sweeper's own exit code fails the Job. + return nil + default: + return ErrMissingTerminalSnapshot + } +} + +const finalReconcileAttempts = 3 + +func (p *Publisher) retryDelay() time.Duration { + if p.PollInterval > 0 { + return p.PollInterval + } + return time.Second +} diff --git a/deploy/operator/internal/dgdrrunpublisher/publisher_test.go b/deploy/operator/internal/dgdrrunpublisher/publisher_test.go new file mode 100644 index 000000000000..290ab10171bb --- /dev/null +++ b/deploy/operator/internal/dgdrrunpublisher/publisher_test.go @@ -0,0 +1,559 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dgdrrunpublisher + +import ( + "context" + "encoding/json" + "errors" + "os" + "path/filepath" + "reflect" + "strings" + "sync" + "testing" + "time" + + "github.com/ai-dynamo/dynamo/deploy/operator/internal/dgdcreconcile" +) + +// fakeCluster records every mutation so tests can assert exactly which writes happened. +type fakeCluster struct { + mu sync.Mutex + candidates map[string]string // name -> id + created map[string]dgdcreconcile.DesiredCandidate // name -> content at creation + creates []string + deletes []string + patches []RunStatus + state SweeperState + patchErr error + createErr error + createCalls int +} + +func newFakeCluster() *fakeCluster { + return &fakeCluster{candidates: map[string]string{}, created: map[string]dgdcreconcile.DesiredCandidate{}} +} + +func (f *fakeCluster) ListCandidates(context.Context) ([]dgdcreconcile.CurrentDGDC, error) { + f.mu.Lock() + defer f.mu.Unlock() + var out []dgdcreconcile.CurrentDGDC + for name, id := range f.candidates { + out = append(out, dgdcreconcile.CurrentDGDC{Name: name, ID: id}) + } + return out, nil +} + +func (f *fakeCluster) CreateCandidate(_ context.Context, name string, c dgdcreconcile.DesiredCandidate) error { + f.mu.Lock() + defer f.mu.Unlock() + f.createCalls++ + if f.createErr != nil { + return f.createErr + } + f.candidates[name] = c.ID + f.created[name] = c + f.creates = append(f.creates, name) + return nil +} + +func (f *fakeCluster) DeleteCandidate(_ context.Context, name string) error { + f.mu.Lock() + defer f.mu.Unlock() + delete(f.candidates, name) + f.deletes = append(f.deletes, name) + return nil +} + +func (f *fakeCluster) PatchRunStatus(_ context.Context, s RunStatus) error { + f.mu.Lock() + defer f.mu.Unlock() + if f.patchErr != nil { + return f.patchErr + } + f.patches = append(f.patches, s) + return nil +} + +func (f *fakeCluster) SweeperState(context.Context) (SweeperState, error) { + f.mu.Lock() + defer f.mu.Unlock() + return f.state, nil +} + +func (f *fakeCluster) setState(s SweeperState) { + f.mu.Lock() + defer f.mu.Unlock() + f.state = s +} + +type cand struct { + id string + failed bool + metrics map[string]any +} + +// snapshotJSON renders a snapshot (JSON is valid YAML, which the real parser accepts). +func snapshotJSON(t *testing.T, phase string, cands ...cand) []byte { + t.Helper() + var list []map[string]any + for _, c := range cands { + entry := map[string]any{"id": c.id, "parameters": map[string]any{"tp": 2}, "metrics": c.metrics} + if c.failed { + entry["outcome"] = OutcomeMaterializationFailed + entry["error"] = "boom" + } else { + entry["outcome"] = OutcomeMaterialized + entry["manifest"] = "kind: DynamoGraphDeployment\n" + } + list = append(list, entry) + } + data, err := json.Marshal(map[string]any{ + "schemaVersion": 1, + "run": map[string]any{"phase": phase, "terminal": phase != PhaseRunning}, + "progress": map[string]any{"round": 3, "evaluated": len(cands)}, + "candidates": list, + }) + if err != nil { + t.Fatal(err) + } + return data +} + +func writeSnapshot(t *testing.T, dir string, data []byte) { + t.Helper() + if err := os.WriteFile(filepath.Join(dir, SnapshotFileName), data, 0o600); err != nil { + t.Fatal(err) + } +} + +func newPublisher(t *testing.T) (*Publisher, *fakeCluster, string) { + t.Helper() + dir := t.TempDir() + fc := newFakeCluster() + return &Publisher{Cluster: fc, SnapshotDir: dir, RunName: "run", PollInterval: time.Millisecond}, fc, dir +} + +func mustParse(t *testing.T, data []byte) *Snapshot { + t.Helper() + snap, err := ParseSnapshot(data) + if err != nil { + t.Fatal(err) + } + return snap +} + +func TestIdentityStableUnderReordering(t *testing.T) { + p, fc, _ := newPublisher(t) + ctx := context.Background() + if err := p.Reconcile(ctx, mustParse(t, snapshotJSON(t, PhaseRunning, cand{id: "a"}, cand{id: "b"}))); err != nil { + t.Fatal(err) + } + creates := len(fc.creates) + if err := p.Reconcile(ctx, mustParse(t, snapshotJSON(t, PhaseRunning, cand{id: "b"}, cand{id: "a"}))); err != nil { + t.Fatal(err) + } + if len(fc.creates) != creates || len(fc.deletes) != 0 { + t.Fatalf("reordering must not create/delete DGDCs: creates=%v deletes=%v", fc.creates, fc.deletes) + } +} + +func TestSwapOrderPatchesOnlyRunStatus(t *testing.T) { + p, fc, _ := newPublisher(t) + ctx := context.Background() + _ = p.Reconcile(ctx, mustParse(t, snapshotJSON(t, PhaseRunning, cand{id: "a"}, cand{id: "b"}))) + _ = p.Reconcile(ctx, mustParse(t, snapshotJSON(t, PhaseRunning, cand{id: "b"}, cand{id: "a"}))) + if len(fc.patches) != 2 { + t.Fatalf("want 2 status patches, got %d", len(fc.patches)) + } + if got, want := fc.patches[1].CandidateNames, []string{"run-b", "run-a"}; !reflect.DeepEqual(got, want) { + t.Fatalf("rank order = %v, want %v", got, want) + } +} + +func TestSwapLeavesBothCandidatesUntouched(t *testing.T) { + p, fc, _ := newPublisher(t) + ctx := context.Background() + if err := p.Reconcile(ctx, mustParse(t, snapshotJSON(t, PhaseRunning, cand{id: "a", metrics: map[string]any{"score": 2.0}}, cand{id: "b", metrics: map[string]any{"score": 1.0}}))); err != nil { + t.Fatal(err) + } + before := map[string]dgdcreconcile.DesiredCandidate{} + for name, content := range fc.created { + before[name] = content + } + if err := p.Reconcile(ctx, mustParse(t, snapshotJSON(t, PhaseRunning, cand{id: "b", metrics: map[string]any{"score": 3.0}}, cand{id: "a", metrics: map[string]any{"score": 2.0}}))); err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(fc.creates, []string{"run-a", "run-b"}) || len(fc.deletes) != 0 { + t.Fatalf("swap touched DGDCs: creates=%v deletes=%v", fc.creates, fc.deletes) + } + if !reflect.DeepEqual(before, fc.created) { + t.Fatalf("a published DGDC's content changed: before=%v after=%v", before, fc.created) + } + if got, want := fc.patches[len(fc.patches)-1].CandidateNames, []string{"run-b", "run-a"}; !reflect.DeepEqual(got, want) { + t.Fatalf("ordered references = %v, want %v", got, want) + } +} + +func TestEvaluatedPointReachesTheClusterComplete(t *testing.T) { + p, fc, _ := newPublisher(t) + if err := p.Reconcile(context.Background(), mustParse(t, snapshotJSON(t, PhaseRunning, cand{id: "a", metrics: map[string]any{"goodputPerGpu": 87.7}}))); err != nil { + t.Fatal(err) + } + got := fc.created["run-a"] + if got.Spec != "kind: DynamoGraphDeployment\n" { + t.Fatalf("manifest = %q", got.Spec) + } + if v, ok := got.Parameters["tp"]; !ok || v != float64(2) { + t.Fatalf("resolved evaluation parameters were lost: %v", got.Parameters) + } + if v, ok := got.Metrics["goodputPerGpu"]; !ok || v != 87.7 { + t.Fatalf("evaluation metrics were lost: %v", got.Metrics) + } +} + +func TestReturningCandidateIsRecreatedIdentically(t *testing.T) { + p, fc, _ := newPublisher(t) + ctx := context.Background() + steps := [][]cand{ + {{id: "a"}, {id: "b", metrics: map[string]any{"score": 1.0}}}, + {{id: "a"}}, + {{id: "a"}, {id: "b", metrics: map[string]any{"score": 1.0}}}, + } + var firstB dgdcreconcile.DesiredCandidate + for i, step := range steps { + if err := p.Reconcile(ctx, mustParse(t, snapshotJSON(t, PhaseRunning, step...))); err != nil { + t.Fatal(err) + } + if i == 0 { + firstB = fc.created["run-b"] + } + } + if !reflect.DeepEqual(fc.creates, []string{"run-a", "run-b", "run-b"}) || !reflect.DeepEqual(fc.deletes, []string{"run-b"}) { + t.Fatalf("creates=%v deletes=%v", fc.creates, fc.deletes) + } + if !reflect.DeepEqual(firstB, fc.created["run-b"]) { + t.Fatalf("recreated candidate differs: %v vs %v", firstB, fc.created["run-b"]) + } +} + +// A published DGDC is immutable: the publisher's view of the cluster has no way to +// update one, so no code path can modify an evaluated point after its initial status. +func TestClusterInterfaceCannotUpdateCandidates(t *testing.T) { + typ := reflect.TypeOf((*Cluster)(nil)).Elem() + var methods []string + for i := 0; i < typ.NumMethod(); i++ { + methods = append(methods, typ.Method(i).Name) + } + want := []string{"CreateCandidate", "DeleteCandidate", "ListCandidates", "PatchRunStatus", "SweeperState"} + if !reflect.DeepEqual(methods, want) { + t.Fatalf("Cluster methods = %v, want %v: adding a candidate-mutating call breaks DGDC immutability", methods, want) + } +} + +func TestRepeatedSnapshotCausesNoWrites(t *testing.T) { + p, fc, _ := newPublisher(t) + ctx := context.Background() + snap := mustParse(t, snapshotJSON(t, PhaseRunning, cand{id: "a"})) + for i := 0; i < 3; i++ { + if err := p.Reconcile(ctx, snap); err != nil { + t.Fatal(err) + } + } + if len(fc.creates) != 1 || len(fc.patches) != 1 || len(fc.deletes) != 0 { + t.Fatalf("creates=%v patches=%d deletes=%v", fc.creates, len(fc.patches), fc.deletes) + } +} + +func TestDroppedCandidateIsDeletedAfterStatusPatch(t *testing.T) { + p, fc, _ := newPublisher(t) + ctx := context.Background() + _ = p.Reconcile(ctx, mustParse(t, snapshotJSON(t, PhaseRunning, cand{id: "a"}, cand{id: "b"}))) + _ = p.Reconcile(ctx, mustParse(t, snapshotJSON(t, PhaseRunning, cand{id: "a"}))) + if !reflect.DeepEqual(fc.deletes, []string{"run-b"}) { + t.Fatalf("deletes = %v", fc.deletes) + } + if got := fc.patches[len(fc.patches)-1].CandidateNames; !reflect.DeepEqual(got, []string{"run-a"}) { + t.Fatalf("status refs = %v", got) + } +} + +func TestFailedMaterializationIsNotCreatedButCounted(t *testing.T) { + p, fc, _ := newPublisher(t) + if err := p.Reconcile(context.Background(), mustParse(t, snapshotJSON(t, PhaseRunning, cand{id: "a"}, cand{id: "x", failed: true}))); err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(fc.creates, []string{"run-a"}) { + t.Fatalf("creates = %v", fc.creates) + } + if got := fc.patches[0].CandidateNames; !reflect.DeepEqual(got, []string{"run-a"}) { + t.Fatalf("refs must not include the failed candidate: %v", got) + } + if msg := fc.patches[0].Message; !strings.Contains(msg, "1 candidate(s) failed materialization") { + t.Fatalf("message = %q", msg) + } +} + +// withTimestamp stamps a snapshot the way the Sweeper container does. +func withTimestamp(t *testing.T, data []byte, timestamp string) []byte { + t.Helper() + var doc map[string]any + if err := json.Unmarshal(data, &doc); err != nil { + t.Fatal(err) + } + doc["timestamp"] = timestamp + out, err := json.Marshal(doc) + if err != nil { + t.Fatal(err) + } + return out +} + +func TestLastProgressComesFromTheSnapshotTimestamp(t *testing.T) { + p, fc, _ := newPublisher(t) + data := withTimestamp(t, snapshotJSON(t, PhaseRunning, cand{id: "a"}), "2026-10-05T21:35:12Z") + if err := p.Reconcile(context.Background(), mustParse(t, data)); err != nil { + t.Fatal(err) + } + want := time.Date(2026, 10, 5, 21, 35, 12, 0, time.UTC) + if got := fc.patches[0].LastProgress; !got.Equal(want) { + t.Fatalf("LastProgress = %v, want %v", got, want) + } +} + +func TestProgressWithoutRankChangeStillPatchesTheProgressTime(t *testing.T) { + p, fc, _ := newPublisher(t) + first := withTimestamp(t, snapshotJSON(t, PhaseRunning, cand{id: "a"}), "2026-10-05T21:35:12Z") + second := withTimestamp(t, snapshotJSON(t, PhaseRunning, cand{id: "a"}), "2026-10-05T21:35:14Z") + for _, data := range [][]byte{first, second, second} { + if err := p.Reconcile(context.Background(), mustParse(t, data)); err != nil { + t.Fatal(err) + } + } + if len(fc.patches) != 2 { + t.Fatalf("want one patch per distinct progress time, got %d", len(fc.patches)) + } + if !reflect.DeepEqual(fc.patches[0].CandidateNames, fc.patches[1].CandidateNames) { + t.Fatalf("refs changed: %v", fc.patches) + } + if len(fc.creates) != 1 { + t.Fatalf("an unchanged candidate must not be created again: %v", fc.creates) + } +} + +func TestMissingOrMalformedTimestampIsNotAnError(t *testing.T) { + for _, timestamp := range []string{"", "yesterday"} { + p, fc, _ := newPublisher(t) + data := withTimestamp(t, snapshotJSON(t, PhaseRunning, cand{id: "a"}), timestamp) + if err := p.Reconcile(context.Background(), mustParse(t, data)); err != nil { + t.Fatalf("timestamp %q: %v", timestamp, err) + } + if got := fc.patches[0].LastProgress; !got.IsZero() { + t.Fatalf("timestamp %q: LastProgress = %v, want zero", timestamp, got) + } + } +} + +func TestCreateFailureDoesNotPatchStatus(t *testing.T) { + p, fc, _ := newPublisher(t) + fc.createErr = errors.New("apiserver down") + if err := p.Reconcile(context.Background(), mustParse(t, snapshotJSON(t, PhaseRunning, cand{id: "a"}))); err == nil { + t.Fatal("want error") + } + if len(fc.patches) != 0 { + t.Fatalf("status must not reference candidates that were not created: %v", fc.patches) + } +} + +func TestParseSnapshotValidation(t *testing.T) { + bad := map[string]string{ + "schema": `{"schemaVersion":2,"run":{"phase":"Running","terminal":false}}`, + "phase": `{"schemaVersion":1,"run":{"phase":"Weird","terminal":false}}`, + "terminal flag": `{"schemaVersion":1,"run":{"phase":"Succeeded","terminal":false}}`, + "duplicate": `{"schemaVersion":1,"run":{"phase":"Running","terminal":false},"candidates":[{"id":"a","outcome":"materialized","manifest":"x"},{"id":"a","outcome":"materialized","manifest":"x"}]}`, + "no manifest": `{"schemaVersion":1,"run":{"phase":"Running","terminal":false},"candidates":[{"id":"a","outcome":"materialized"}]}`, + "no error": `{"schemaVersion":1,"run":{"phase":"Running","terminal":false},"candidates":[{"id":"a","outcome":"materialization_failed"}]}`, + "empty id": `{"schemaVersion":1,"run":{"phase":"Running","terminal":false},"candidates":[{"id":"","outcome":"materialized","manifest":"x"}]}`, + } + for name, doc := range bad { + if _, err := ParseSnapshot([]byte(doc)); err == nil { + t.Errorf("%s: want error", name) + } + } +} + +func runWithTimeout(t *testing.T, p *Publisher) error { + t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + return p.Run(ctx) +} + +func lastPatch(t *testing.T, fc *fakeCluster) RunStatus { + t.Helper() + if len(fc.patches) == 0 { + t.Fatal("no run status patch was written") + } + return fc.patches[len(fc.patches)-1] +} + +// Orderly completion: terminal snapshot is reconciled and the publisher exits 0 even +// though the Sweeper container is still shutting down. +func TestRunTerminalSnapshotAcknowledges(t *testing.T) { + p, fc, dir := newPublisher(t) + writeSnapshot(t, dir, snapshotJSON(t, PhaseSucceeded, cand{id: "a"})) + err := runWithTimeout(t, p) + if err != nil || ExitCode(err) != ExitAcknowledged { + t.Fatalf("err = %v", err) + } + if got := lastPatch(t, fc).CandidateNames; !reflect.DeepEqual(got, []string{"run-a"}) { + t.Fatalf("refs = %v", got) + } +} + +// Caught Sweeper failure: a terminal Failed snapshot still has its projection reconciled, +// and the publisher succeeds (the Sweeper's exit code fails the Job). +func TestRunTerminalFailedSnapshotIsReconciled(t *testing.T) { + p, fc, dir := newPublisher(t) + writeSnapshot(t, dir, snapshotJSON(t, PhaseFailed, cand{id: "a"})) + fc.setState(SweeperState{Exited: true, ExitCode: 1}) + if err := runWithTimeout(t, p); err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(fc.creates, []string{"run-a"}) { + t.Fatalf("creates = %v", fc.creates) + } +} + +func TestRunReconcilesTerminalSnapshotWrittenAfterRunningOne(t *testing.T) { + p, fc, dir := newPublisher(t) + writeSnapshot(t, dir, snapshotJSON(t, PhaseRunning, cand{id: "a"})) + done := make(chan error, 1) + go func() { done <- runWithTimeout(t, p) }() + time.Sleep(20 * time.Millisecond) + writeSnapshot(t, dir, snapshotJSON(t, PhaseSucceeded, cand{id: "a"}, cand{id: "b"})) + if err := <-done; err != nil { + t.Fatal(err) + } + if got := lastPatch(t, fc).CandidateNames; len(got) != 2 { + t.Fatalf("terminal projection lost: %v", got) + } +} + +// Abrupt non-zero exit without a terminal snapshot: the last snapshot is reconciled and +// the publisher exits 0. +func TestRunCrashWithoutTerminalSnapshotReconcilesLastSnapshot(t *testing.T) { + p, fc, dir := newPublisher(t) + writeSnapshot(t, dir, snapshotJSON(t, PhaseRunning, cand{id: "a"})) + fc.setState(SweeperState{Exited: true, ExitCode: 137}) + if err := runWithTimeout(t, p); err != nil { + t.Fatal(err) + } + if got := lastPatch(t, fc).CandidateNames; !reflect.DeepEqual(got, []string{"run-a"}) { + t.Fatalf("refs = %v", got) + } +} + +// Nothing was ever written and the Sweeper crashed: nothing to reconcile, exit 0. +func TestRunCrashBeforeAnySnapshot(t *testing.T) { + p, fc, _ := newPublisher(t) + fc.setState(SweeperState{Exited: true, ExitCode: 137}) + if err := runWithTimeout(t, p); err != nil { + t.Fatal(err) + } + if len(fc.creates) != 0 || len(fc.patches) != 0 { + t.Fatalf("unexpected writes: %v %v", fc.creates, fc.patches) + } +} + +// Exit 0 without a terminal snapshot is a protocol violation: the last snapshot is still +// reconciled but the publisher fails with the dedicated exit code. +func TestRunCleanExitWithoutTerminalSnapshotIsProtocolViolation(t *testing.T) { + p, fc, dir := newPublisher(t) + writeSnapshot(t, dir, snapshotJSON(t, PhaseRunning, cand{id: "a"})) + fc.setState(SweeperState{Exited: true, ExitCode: 0}) + err := runWithTimeout(t, p) + if !errors.Is(err, ErrMissingTerminalSnapshot) || ExitCode(err) != ExitMissingTerminalSnapshot { + t.Fatalf("err = %v", err) + } + if !reflect.DeepEqual(fc.creates, []string{"run-a"}) { + t.Fatalf("last snapshot must still be reconciled: %v", fc.creates) + } +} + +func TestRunCleanExitWithNoSnapshotAtAllIsProtocolViolation(t *testing.T) { + p, fc, _ := newPublisher(t) + fc.setState(SweeperState{Exited: true, ExitCode: 0}) + if err := runWithTimeout(t, p); !errors.Is(err, ErrMissingTerminalSnapshot) { + t.Fatalf("err = %v", err) + } +} + +func TestRunFinalReconcileFailureFails(t *testing.T) { + p, fc, dir := newPublisher(t) + writeSnapshot(t, dir, snapshotJSON(t, PhaseRunning, cand{id: "a"})) + fc.createErr = errors.New("apiserver down") + fc.setState(SweeperState{Exited: true, ExitCode: 1}) + err := runWithTimeout(t, p) + if err == nil || ExitCode(err) != ExitReconcileFailed { + t.Fatalf("err = %v", err) + } + if fc.createCalls < finalReconcileAttempts { + t.Fatalf("expected %d attempts, got %d", finalReconcileAttempts, fc.createCalls) + } +} + +func TestRunTerminalSnapshotReconcileFailureIsRetriedUntilSuccess(t *testing.T) { + p, fc, dir := newPublisher(t) + writeSnapshot(t, dir, snapshotJSON(t, PhaseSucceeded, cand{id: "a"})) + fc.createErr = errors.New("flaky") + done := make(chan error, 1) + go func() { done <- runWithTimeout(t, p) }() + time.Sleep(20 * time.Millisecond) + fc.mu.Lock() + fc.createErr = nil + fc.mu.Unlock() + if err := <-done; err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(fc.creates, []string{"run-a"}) { + t.Fatalf("creates = %v", fc.creates) + } +} + +func TestRunMalformedProjectionFailsImmediately(t *testing.T) { + p, fc, dir := newPublisher(t) + fc.candidates["x"] = "dup" + fc.candidates["y"] = "dup" + writeSnapshot(t, dir, snapshotJSON(t, PhaseRunning, cand{id: "a"})) + err := runWithTimeout(t, p) + var inputErr *dgdcreconcile.DiffInputError + if !errors.As(err, &inputErr) { + t.Fatalf("err = %v", err) + } +} + +func TestRunContextCancelledReturnsError(t *testing.T) { + p, _, _ := newPublisher(t) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if err := p.Run(ctx); !errors.Is(err, context.Canceled) { + t.Fatalf("err = %v", err) + } +} diff --git a/deploy/operator/internal/dgdrrunpublisher/snapshot.go b/deploy/operator/internal/dgdrrunpublisher/snapshot.go new file mode 100644 index 000000000000..3d53f13dfb77 --- /dev/null +++ b/deploy/operator/internal/dgdrrunpublisher/snapshot.go @@ -0,0 +1,118 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dgdrrunpublisher + +import ( + "fmt" + + "sigs.k8s.io/yaml" +) + +// SnapshotSchemaVersion is the only wire version this publisher understands. +const SnapshotSchemaVersion = 1 + +// Candidate outcomes carried by the snapshot. +const ( + OutcomeMaterialized = "materialized" + OutcomeMaterializationFailed = "materialization_failed" +) + +// Run phases carried by the snapshot. +const ( + PhaseRunning = "Running" + PhaseSucceeded = "Succeeded" + PhaseFailed = "Failed" +) + +// Snapshot is the complete current projection of one run, written atomically by the +// Sweeper container (see dynamo.aisimulate.output.dgdr_run.snapshot). It is desired +// state, not a command list: each snapshot supersedes the previous one entirely. +type Snapshot struct { + SchemaVersion int `json:"schemaVersion"` + Timestamp string `json:"timestamp,omitempty"` + Run SnapshotRun `json:"run"` + Progress SnapshotProgress `json:"progress"` + Candidates []SnapshotCandidate `json:"candidates"` +} + +// SnapshotRun is the run-level state. +type SnapshotRun struct { + Phase string `json:"phase"` + Terminal bool `json:"terminal"` + Message string `json:"message,omitempty"` + Error string `json:"error,omitempty"` +} + +// SnapshotProgress is Sweeper progress. +type SnapshotProgress struct { + Round int32 `json:"round"` + Evaluated int32 `json:"evaluated"` +} + +// SnapshotCandidate is one evaluated point. Candidates are ordered best-first for +// scalar searches; the id never depends on that order. +type SnapshotCandidate struct { + ID string `json:"id"` + Outcome string `json:"outcome"` + Parameters map[string]any `json:"parameters,omitempty"` + Metrics map[string]any `json:"metrics,omitempty"` + Manifest string `json:"manifest,omitempty"` + Error string `json:"error,omitempty"` +} + +// ParseSnapshot decodes and validates snapshot bytes. +func ParseSnapshot(data []byte) (*Snapshot, error) { + var snap Snapshot + if err := yaml.Unmarshal(data, &snap); err != nil { + return nil, fmt.Errorf("decoding snapshot: %w", err) + } + if snap.SchemaVersion != SnapshotSchemaVersion { + return nil, fmt.Errorf("unsupported snapshot schemaVersion %d (want %d)", snap.SchemaVersion, SnapshotSchemaVersion) + } + switch snap.Run.Phase { + case PhaseRunning, PhaseSucceeded, PhaseFailed: + default: + return nil, fmt.Errorf("unknown run phase %q", snap.Run.Phase) + } + if snap.Run.Terminal != (snap.Run.Phase != PhaseRunning) { + return nil, fmt.Errorf("run phase %q inconsistent with terminal=%t", snap.Run.Phase, snap.Run.Terminal) + } + seen := make(map[string]struct{}, len(snap.Candidates)) + for i, candidate := range snap.Candidates { + if candidate.ID == "" { + return nil, fmt.Errorf("candidate %d has an empty id", i) + } + if _, dup := seen[candidate.ID]; dup { + return nil, fmt.Errorf("duplicate candidate id %q", candidate.ID) + } + seen[candidate.ID] = struct{}{} + switch candidate.Outcome { + case OutcomeMaterialized: + if candidate.Manifest == "" { + return nil, fmt.Errorf("materialized candidate %q has no manifest", candidate.ID) + } + case OutcomeMaterializationFailed: + if candidate.Error == "" { + return nil, fmt.Errorf("failed candidate %q has no error", candidate.ID) + } + default: + return nil, fmt.Errorf("candidate %q has unknown outcome %q", candidate.ID, candidate.Outcome) + } + } + return &snap, nil +} From 35f30859969f3eade137ee43587a242e9a85a848 Mon Sep 17 00:00:00 2001 From: devivasudevan <49675305+devivasudevan@users.noreply.github.com> Date: Wed, 7 Oct 2026 15:46:05 +0200 Subject: [PATCH 02/40] fix(operator): label-safe run selector, fail on terminal Failed snapshot; trim redundant tests/comments --- .../internal/dgdrrunpublisher/kube.go | 23 +++++- .../internal/dgdrrunpublisher/publisher.go | 25 ++++++- .../dgdrrunpublisher/publisher_test.go | 75 +++++++++---------- .../internal/dgdrrunpublisher/snapshot.go | 3 - 4 files changed, 78 insertions(+), 48 deletions(-) diff --git a/deploy/operator/internal/dgdrrunpublisher/kube.go b/deploy/operator/internal/dgdrrunpublisher/kube.go index 4f3d3913f2e3..36a7ee79a5e9 100644 --- a/deploy/operator/internal/dgdrrunpublisher/kube.go +++ b/deploy/operator/internal/dgdrrunpublisher/kube.go @@ -19,8 +19,11 @@ package dgdrrunpublisher import ( "context" + "crypto/sha256" + "encoding/hex" "encoding/json" "fmt" + "strings" v1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" "github.com/ai-dynamo/dynamo/deploy/operator/internal/dgdcreconcile" @@ -43,6 +46,22 @@ const ( LabelCandidateID = "nvidia.com/dgdr-candidate-id" ) +// maxLabelValue is the Kubernetes limit on a label value; run names may be up to 253. +const maxLabelValue = 63 + +// RunLabelValue is the label-safe, stable selector value for a run name. Names that +// fit are used as-is; longer ones are truncated and suffixed with a hash of the full +// name so they stay distinct. Every reader and writer of LabelRunName must use it. +func RunLabelValue(runName string) string { + if len(runName) <= maxLabelValue { + return runName + } + sum := sha256.Sum256([]byte(runName)) + suffix := hex.EncodeToString(sum[:8]) + prefix := strings.TrimRight(runName[:maxLabelValue-len(suffix)-1], "-_.") + return prefix + "-" + suffix +} + // KubeCluster is the Cluster implementation backed by a controller-runtime client. It // should be a direct (uncached) client: the publisher is a short-lived sidecar, so it // neither needs nor wants an informer cache. @@ -63,7 +82,7 @@ var _ Cluster = (*KubeCluster)(nil) func (k *KubeCluster) ListCandidates(ctx context.Context) ([]dgdcreconcile.CurrentDGDC, error) { var list v1beta2.DynamoGraphDeploymentCandidateList - if err := k.Client.List(ctx, &list, client.InNamespace(k.Namespace), client.MatchingLabels{LabelRunName: k.RunName}); err != nil { + if err := k.Client.List(ctx, &list, client.InNamespace(k.Namespace), client.MatchingLabels{LabelRunName: RunLabelValue(k.RunName)}); err != nil { return nil, err } out := make([]dgdcreconcile.CurrentDGDC, 0, len(list.Items)) @@ -97,7 +116,7 @@ func (k *KubeCluster) CreateCandidate(ctx context.Context, name string, candidat ObjectMeta: metav1.ObjectMeta{ Namespace: k.Namespace, Name: name, - Labels: map[string]string{LabelRunName: k.RunName, LabelCandidateID: candidate.ID}, + Labels: map[string]string{LabelRunName: RunLabelValue(k.RunName), LabelCandidateID: candidate.ID}, }, Spec: v1beta2.DynamoGraphDeploymentCandidateSpec{ DynamoGraphDeploymentSpec: doc.Spec, diff --git a/deploy/operator/internal/dgdrrunpublisher/publisher.go b/deploy/operator/internal/dgdrrunpublisher/publisher.go index b8694002a32f..9e347a49a31e 100644 --- a/deploy/operator/internal/dgdrrunpublisher/publisher.go +++ b/deploy/operator/internal/dgdrrunpublisher/publisher.go @@ -29,7 +29,8 @@ // // Termination state machine (see Publisher.Run): // -// terminal snapshot reconciled -> exit 0 +// terminal Succeeded snapshot reconciled -> exit 0 +// terminal Failed snapshot reconciled -> exit ExitRunFailed // Sweeper exited non-zero, no terminal snapshot -> reconcile last snapshot, exit 0 // (the Sweeper's exit fails the Job) // Sweeper exited zero, no terminal snapshot -> reconcile last snapshot, @@ -55,12 +56,18 @@ const ( ExitAcknowledged = 0 ExitReconcileFailed = 1 ExitMissingTerminalSnapshot = 3 + ExitRunFailed = 4 ) // ErrMissingTerminalSnapshot means the Sweeper exited successfully without publishing a // terminal snapshot, which violates the producer protocol. var ErrMissingTerminalSnapshot = errors.New("sweeper exited without publishing a terminal snapshot") +// ErrRunFailed means the Sweeper published a terminal Failed snapshot. The snapshot is +// reconciled first; the error keeps a producer that reports failure but exits 0 from +// letting the Job complete successfully. +var ErrRunFailed = errors.New("sweeper reported a failed run") + // ExitCode maps the result of Run to the process exit code. func ExitCode(err error) int { switch { @@ -68,6 +75,8 @@ func ExitCode(err error) int { return ExitAcknowledged case errors.Is(err, ErrMissingTerminalSnapshot): return ExitMissingTerminalSnapshot + case errors.Is(err, ErrRunFailed): + return ExitRunFailed default: return ExitReconcileFailed } @@ -116,6 +125,7 @@ type Publisher struct { lastStatus *RunStatus haveSnapshot bool // a snapshot has been reconciled terminal bool // the reconciled snapshot was terminal + phase string } // CandidateName is the DGDC name for a candidate id. It depends only on the run and the @@ -242,6 +252,7 @@ func (p *Publisher) syncOnce(ctx context.Context) error { p.lastRaw = data p.haveSnapshot = true p.terminal = snap.Run.Terminal + p.phase = snap.Run.Phase return nil } @@ -268,7 +279,7 @@ func (p *Publisher) Run(ctx context.Context) error { } if syncErr == nil && p.terminal { - return nil + return p.terminalResult() } if stateErr == nil && state.Exited { return p.finish(ctx, state, syncErr) @@ -282,6 +293,14 @@ func (p *Publisher) Run(ctx context.Context) error { } } +// terminalResult is the outcome once a terminal snapshot has been reconciled. +func (p *Publisher) terminalResult() error { + if p.phase == PhaseFailed { + return ErrRunFailed + } + return nil +} + // finish handles the Sweeper having exited without a reconciled terminal snapshot. Any // snapshot it ever wrote is now visible, so the last one is reconciled (with retries, // since nothing will rewrite it) before deciding the outcome. @@ -299,7 +318,7 @@ func (p *Publisher) finish(ctx context.Context, state SweeperState, syncErr erro } switch { case p.terminal: - return nil + return p.terminalResult() case state.ExitCode != 0: // Crash or caught failure without a terminal snapshot: the last-known projection // is reconciled above; the Sweeper's own exit code fails the Job. diff --git a/deploy/operator/internal/dgdrrunpublisher/publisher_test.go b/deploy/operator/internal/dgdrrunpublisher/publisher_test.go index 290ab10171bb..a9e78cc90eac 100644 --- a/deploy/operator/internal/dgdrrunpublisher/publisher_test.go +++ b/deploy/operator/internal/dgdrrunpublisher/publisher_test.go @@ -159,34 +159,6 @@ func mustParse(t *testing.T, data []byte) *Snapshot { return snap } -func TestIdentityStableUnderReordering(t *testing.T) { - p, fc, _ := newPublisher(t) - ctx := context.Background() - if err := p.Reconcile(ctx, mustParse(t, snapshotJSON(t, PhaseRunning, cand{id: "a"}, cand{id: "b"}))); err != nil { - t.Fatal(err) - } - creates := len(fc.creates) - if err := p.Reconcile(ctx, mustParse(t, snapshotJSON(t, PhaseRunning, cand{id: "b"}, cand{id: "a"}))); err != nil { - t.Fatal(err) - } - if len(fc.creates) != creates || len(fc.deletes) != 0 { - t.Fatalf("reordering must not create/delete DGDCs: creates=%v deletes=%v", fc.creates, fc.deletes) - } -} - -func TestSwapOrderPatchesOnlyRunStatus(t *testing.T) { - p, fc, _ := newPublisher(t) - ctx := context.Background() - _ = p.Reconcile(ctx, mustParse(t, snapshotJSON(t, PhaseRunning, cand{id: "a"}, cand{id: "b"}))) - _ = p.Reconcile(ctx, mustParse(t, snapshotJSON(t, PhaseRunning, cand{id: "b"}, cand{id: "a"}))) - if len(fc.patches) != 2 { - t.Fatalf("want 2 status patches, got %d", len(fc.patches)) - } - if got, want := fc.patches[1].CandidateNames, []string{"run-b", "run-a"}; !reflect.DeepEqual(got, want) { - t.Fatalf("rank order = %v, want %v", got, want) - } -} - func TestSwapLeavesBothCandidatesUntouched(t *testing.T) { p, fc, _ := newPublisher(t) ctx := context.Background() @@ -310,7 +282,6 @@ func TestFailedMaterializationIsNotCreatedButCounted(t *testing.T) { } } -// withTimestamp stamps a snapshot the way the Sweeper container does. func withTimestamp(t *testing.T, data []byte, timestamp string) []byte { t.Helper() var doc map[string]any @@ -427,17 +398,41 @@ func TestRunTerminalSnapshotAcknowledges(t *testing.T) { } } -// Caught Sweeper failure: a terminal Failed snapshot still has its projection reconciled, -// and the publisher succeeds (the Sweeper's exit code fails the Job). -func TestRunTerminalFailedSnapshotIsReconciled(t *testing.T) { - p, fc, dir := newPublisher(t) - writeSnapshot(t, dir, snapshotJSON(t, PhaseFailed, cand{id: "a"})) - fc.setState(SweeperState{Exited: true, ExitCode: 1}) - if err := runWithTimeout(t, p); err != nil { - t.Fatal(err) - } - if !reflect.DeepEqual(fc.creates, []string{"run-a"}) { - t.Fatalf("creates = %v", fc.creates) +// A terminal Failed snapshot has its projection reconciled, then fails the publisher +// whether or not the Sweeper itself exited non-zero. +func TestRunTerminalFailedSnapshotIsReconciledThenFails(t *testing.T) { + for name, state := range map[string]SweeperState{ + "sweeper exit 0": {Exited: true, ExitCode: 0}, + "sweeper exit 1": {Exited: true, ExitCode: 1}, + "still running": {}, + } { + t.Run(name, func(t *testing.T) { + p, fc, dir := newPublisher(t) + writeSnapshot(t, dir, snapshotJSON(t, PhaseFailed, cand{id: "a"})) + fc.setState(state) + err := runWithTimeout(t, p) + if !errors.Is(err, ErrRunFailed) || ExitCode(err) != ExitRunFailed { + t.Fatalf("err = %v", err) + } + if !reflect.DeepEqual(fc.creates, []string{"run-a"}) { + t.Fatalf("creates = %v", fc.creates) + } + }) + } +} + +func TestRunLabelValueIsLabelSafeAndStable(t *testing.T) { + if got := RunLabelValue("short-run"); got != "short-run" { + t.Fatalf("short names are kept: %q", got) + } + long := strings.Repeat("a", 100) + "-x" + other := strings.Repeat("a", 100) + "-y" + got := RunLabelValue(long) + if len(got) > 63 || got == RunLabelValue(other) || got != RunLabelValue(long) { + t.Fatalf("label = %q (%d)", got, len(got)) + } + if last := got[len(got)-1]; !(last >= '0' && last <= '9' || last >= 'a' && last <= 'f') { + t.Fatalf("must end alphanumeric: %q", got) } } diff --git a/deploy/operator/internal/dgdrrunpublisher/snapshot.go b/deploy/operator/internal/dgdrrunpublisher/snapshot.go index 3d53f13dfb77..daf3e14b545e 100644 --- a/deploy/operator/internal/dgdrrunpublisher/snapshot.go +++ b/deploy/operator/internal/dgdrrunpublisher/snapshot.go @@ -50,7 +50,6 @@ type Snapshot struct { Candidates []SnapshotCandidate `json:"candidates"` } -// SnapshotRun is the run-level state. type SnapshotRun struct { Phase string `json:"phase"` Terminal bool `json:"terminal"` @@ -58,7 +57,6 @@ type SnapshotRun struct { Error string `json:"error,omitempty"` } -// SnapshotProgress is Sweeper progress. type SnapshotProgress struct { Round int32 `json:"round"` Evaluated int32 `json:"evaluated"` @@ -75,7 +73,6 @@ type SnapshotCandidate struct { Error string `json:"error,omitempty"` } -// ParseSnapshot decodes and validates snapshot bytes. func ParseSnapshot(data []byte) (*Snapshot, error) { var snap Snapshot if err := yaml.Unmarshal(data, &snap); err != nil { From 14160a21d7fc9ea200b016d06b25aa8bcb8542e9 Mon Sep 17 00:00:00 2001 From: devivasudevan <49675305+devivasudevan@users.noreply.github.com> Date: Thu, 8 Oct 2026 13:00:59 +0200 Subject: [PATCH 03/40] fix(operator): scope candidates to run UID, bound names, validate ids/manifests, reject regressing snapshots --- .../operator/internal/dgdcreconcile/diff.go | 9 +- .../internal/dgdcreconcile/diff_test.go | 1 + .../internal/dgdrrunpublisher/kube.go | 96 ++++++---- .../internal/dgdrrunpublisher/publisher.go | 71 +++++--- .../dgdrrunpublisher/publisher_test.go | 168 +++++++++++++----- .../internal/dgdrrunpublisher/snapshot.go | 57 +++++- 6 files changed, 294 insertions(+), 108 deletions(-) diff --git a/deploy/operator/internal/dgdcreconcile/diff.go b/deploy/operator/internal/dgdcreconcile/diff.go index 6120b8a4ecc4..26e48c2e23ed 100644 --- a/deploy/operator/internal/dgdcreconcile/diff.go +++ b/deploy/operator/internal/dgdcreconcile/diff.go @@ -61,10 +61,15 @@ type DesiredCandidate struct { type CurrentDGDC struct { Name string ID string + // Incomplete means the DGDC exists but its initial status was never populated (for + // example the process stopped between creating it and writing its status). + Incomplete bool } // Actions is what the caller must apply. There are no updates: DGDCs are immutable. type Actions struct { + // Creates are idempotent "ensure" calls: they also cover a desired candidate whose + // DGDC exists but is Incomplete, so its status is finished before it is referenced. Creates []DesiredCandidate Deletes []string // DGDC names } @@ -83,6 +88,7 @@ func ComputeActions(desired []DesiredCandidate, current []CurrentDGDC) (Actions, desiredIDs[candidate.ID] = struct{}{} } currentIDs := make(map[string]struct{}, len(current)) + incomplete := make(map[string]bool, len(current)) for _, existing := range current { if existing.ID == "" { return Actions{}, newDiffInputError("DGDC %s has no candidate id", existing.Name) @@ -91,11 +97,12 @@ func ComputeActions(desired []DesiredCandidate, current []CurrentDGDC) (Actions, return Actions{}, newDiffInputError("duplicate id in current set: %s", existing.ID) } currentIDs[existing.ID] = struct{}{} + incomplete[existing.ID] = existing.Incomplete } var actions Actions for _, candidate := range desired { - if _, exists := currentIDs[candidate.ID]; !exists { + if _, exists := currentIDs[candidate.ID]; !exists || incomplete[candidate.ID] { actions.Creates = append(actions.Creates, candidate) } } diff --git a/deploy/operator/internal/dgdcreconcile/diff_test.go b/deploy/operator/internal/dgdcreconcile/diff_test.go index 6a5358b0a399..3d5eda01441c 100644 --- a/deploy/operator/internal/dgdcreconcile/diff_test.go +++ b/deploy/operator/internal/dgdcreconcile/diff_test.go @@ -60,6 +60,7 @@ func TestComputeActions(t *testing.T) { {"all gone", nil, c("n-a", "a", "n-b", "b"), nil, []string{"n-a", "n-b"}}, {"steady state needs no action", d("a", "b"), c("n-a", "a", "n-b", "b"), nil, nil}, {"reordering desired needs no action", d("b", "a"), c("n-a", "a", "n-b", "b"), nil, nil}, + {"incomplete DGDC is ensured again", d("a", "b"), []CurrentDGDC{{Name: "n-a", ID: "a"}, {Name: "n-b", ID: "b", Incomplete: true}}, []string{"b"}, nil}, {"mixed", d("a", "c"), c("n-a", "a", "n-b", "b"), []string{"c"}, []string{"n-b"}}, } for _, tc := range tests { diff --git a/deploy/operator/internal/dgdrrunpublisher/kube.go b/deploy/operator/internal/dgdrrunpublisher/kube.go index 36a7ee79a5e9..31355f781ad1 100644 --- a/deploy/operator/internal/dgdrrunpublisher/kube.go +++ b/deploy/operator/internal/dgdrrunpublisher/kube.go @@ -19,11 +19,8 @@ package dgdrrunpublisher import ( "context" - "crypto/sha256" - "encoding/hex" "encoding/json" "fmt" - "strings" v1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" "github.com/ai-dynamo/dynamo/deploy/operator/internal/dgdcreconcile" @@ -40,27 +37,16 @@ import ( ) const ( - // LabelRunName ties a candidate to its run. - LabelRunName = "nvidia.com/dgdr-run-name" + // LabelRunUID ties a candidate to one incarnation of its run. The run name can be + // reused after a delete and recreate; the UID cannot, so candidates left over from a + // previous run of the same name are never mistaken for current ones. + LabelRunUID = "nvidia.com/dgdr-run-uid" // LabelCandidateID records the stable candidate id (the identity read back on list). LabelCandidateID = "nvidia.com/dgdr-candidate-id" -) - -// maxLabelValue is the Kubernetes limit on a label value; run names may be up to 253. -const maxLabelValue = 63 -// RunLabelValue is the label-safe, stable selector value for a run name. Names that -// fit are used as-is; longer ones are truncated and suffixed with a hash of the full -// name so they stay distinct. Every reader and writer of LabelRunName must use it. -func RunLabelValue(runName string) string { - if len(runName) <= maxLabelValue { - return runName - } - sum := sha256.Sum256([]byte(runName)) - suffix := hex.EncodeToString(sum[:8]) - prefix := strings.TrimRight(runName[:maxLabelValue-len(suffix)-1], "-_.") - return prefix + "-" + suffix -} + evaluatedCondition = "Evaluated" + dgdKind = "DynamoGraphDeployment" +) // KubeCluster is the Cluster implementation backed by a controller-runtime client. It // should be a direct (uncached) client: the publisher is a short-lived sidecar, so it @@ -80,33 +66,64 @@ type KubeCluster struct { var _ Cluster = (*KubeCluster)(nil) +func (k *KubeCluster) getRun(ctx context.Context) (*v1beta2.DynamoGraphDeploymentRun, error) { + var run v1beta2.DynamoGraphDeploymentRun + if err := k.Client.Get(ctx, types.NamespacedName{Namespace: k.Namespace, Name: k.RunName}, &run); err != nil { + return nil, fmt.Errorf("getting run: %w", err) + } + return &run, nil +} + func (k *KubeCluster) ListCandidates(ctx context.Context) ([]dgdcreconcile.CurrentDGDC, error) { + run, err := k.getRun(ctx) + if err != nil { + return nil, err + } var list v1beta2.DynamoGraphDeploymentCandidateList - if err := k.Client.List(ctx, &list, client.InNamespace(k.Namespace), client.MatchingLabels{LabelRunName: RunLabelValue(k.RunName)}); err != nil { + if err := k.Client.List(ctx, &list, client.InNamespace(k.Namespace), client.MatchingLabels{LabelRunUID: string(run.UID)}); err != nil { return nil, err } out := make([]dgdcreconcile.CurrentDGDC, 0, len(list.Items)) for i := range list.Items { out = append(out, dgdcreconcile.CurrentDGDC{ - Name: list.Items[i].Name, - ID: list.Items[i].Labels[LabelCandidateID], + Name: list.Items[i].Name, + ID: list.Items[i].Labels[LabelCandidateID], + Incomplete: !meta.IsStatusConditionTrue(list.Items[i].Status.Conditions, evaluatedCondition), }) } return out, nil } -// CreateCandidate creates the candidate and populates its status exactly once. Both -// steps are idempotent so a retry after a partial failure converges. -func (k *KubeCluster) CreateCandidate(ctx context.Context, name string, candidate dgdcreconcile.DesiredCandidate) error { +// decodeSpec extracts the DGD spec from a rendered manifest, rejecting anything that is +// not a DynamoGraphDeployment with a spec rather than creating an empty candidate. +func decodeSpec(candidate dgdcreconcile.DesiredCandidate) (*v1beta1.DynamoGraphDeploymentSpec, error) { var doc struct { - Spec v1beta1.DynamoGraphDeploymentSpec `json:"spec"` + Kind string `json:"kind"` + Spec *v1beta1.DynamoGraphDeploymentSpec `json:"spec"` } if err := yaml.Unmarshal([]byte(candidate.Spec), &doc); err != nil { - return fmt.Errorf("decoding manifest of candidate %s: %w", candidate.ID, err) + return nil, fmt.Errorf("decoding manifest of candidate %s: %w", candidate.ID, err) } - var run v1beta2.DynamoGraphDeploymentRun - if err := k.Client.Get(ctx, types.NamespacedName{Namespace: k.Namespace, Name: k.RunName}, &run); err != nil { - return fmt.Errorf("getting run: %w", err) + if doc.Kind != dgdKind { + return nil, fmt.Errorf("manifest of candidate %s is a %q, want %s", candidate.ID, doc.Kind, dgdKind) + } + if doc.Spec == nil { + return nil, fmt.Errorf("manifest of candidate %s has no spec", candidate.ID) + } + return doc.Spec, nil +} + +// CreateCandidate creates the candidate and populates its status exactly once. Both +// steps are idempotent so a retry after a partial failure converges; the publisher +// calls it again for a candidate whose status was never populated. +func (k *KubeCluster) CreateCandidate(ctx context.Context, name string, candidate dgdcreconcile.DesiredCandidate) error { + spec, err := decodeSpec(candidate) + if err != nil { + return err + } + run, err := k.getRun(ctx) + if err != nil { + return err } parameters, err := rawExtension(candidate.Parameters) if err != nil { @@ -116,14 +133,14 @@ func (k *KubeCluster) CreateCandidate(ctx context.Context, name string, candidat ObjectMeta: metav1.ObjectMeta{ Namespace: k.Namespace, Name: name, - Labels: map[string]string{LabelRunName: RunLabelValue(k.RunName), LabelCandidateID: candidate.ID}, + Labels: map[string]string{LabelRunUID: string(run.UID), LabelCandidateID: candidate.ID}, }, Spec: v1beta2.DynamoGraphDeploymentCandidateSpec{ - DynamoGraphDeploymentSpec: doc.Spec, + DynamoGraphDeploymentSpec: *spec, Parameters: parameters, }, } - if err := controllerutil.SetControllerReference(&run, object, k.Client.Scheme()); err != nil { + if err := controllerutil.SetControllerReference(run, object, k.Client.Scheme()); err != nil { return err } if err := k.Client.Create(ctx, object); err != nil && !apierrors.IsAlreadyExists(err) { @@ -134,7 +151,10 @@ func (k *KubeCluster) CreateCandidate(ctx context.Context, name string, candidat if err := k.Client.Get(ctx, types.NamespacedName{Namespace: k.Namespace, Name: name}, &stored); err != nil { return err } - if len(stored.Status.Conditions) > 0 { + if !metav1.IsControlledBy(&stored, run) { + return fmt.Errorf("candidate %s already exists and is not controlled by run %s (uid %s)", name, k.RunName, run.UID) + } + if meta.IsStatusConditionTrue(stored.Status.Conditions, evaluatedCondition) { return nil // status already populated; candidates are immutable afterwards } metrics, err := rawExtension(candidate.Metrics) @@ -143,9 +163,9 @@ func (k *KubeCluster) CreateCandidate(ctx context.Context, name string, candidat } stored.Status.Metrics = metrics meta.SetStatusCondition(&stored.Status.Conditions, metav1.Condition{ - Type: "Evaluated", + Type: evaluatedCondition, Status: metav1.ConditionTrue, - Reason: "Evaluated", + Reason: evaluatedCondition, Message: "evaluated by the Sweeper", }) return k.Client.Status().Update(ctx, &stored) diff --git a/deploy/operator/internal/dgdrrunpublisher/publisher.go b/deploy/operator/internal/dgdrrunpublisher/publisher.go index 9e347a49a31e..213f5cb15551 100644 --- a/deploy/operator/internal/dgdrrunpublisher/publisher.go +++ b/deploy/operator/internal/dgdrrunpublisher/publisher.go @@ -34,18 +34,22 @@ // Sweeper exited non-zero, no terminal snapshot -> reconcile last snapshot, exit 0 // (the Sweeper's exit fails the Job) // Sweeper exited zero, no terminal snapshot -> reconcile last snapshot, -// exit ExitMissingTerminalSnapshot +// exit ExitProtocolViolation +// invalid or regressing snapshot, Sweeper exited -> exit ExitProtocolViolation // final/last snapshot cannot be reconciled -> exit ExitReconcileFailed package dgdrrunpublisher import ( "context" + "crypto/sha256" + "encoding/hex" "errors" "fmt" "log" "os" "path/filepath" "reflect" + "strings" "time" "github.com/ai-dynamo/dynamo/deploy/operator/internal/dgdcreconcile" @@ -53,15 +57,15 @@ import ( // Process exit codes of the publisher binary. const ( - ExitAcknowledged = 0 - ExitReconcileFailed = 1 - ExitMissingTerminalSnapshot = 3 - ExitRunFailed = 4 + ExitAcknowledged = 0 + ExitReconcileFailed = 1 + ExitProtocolViolation = 3 + ExitRunFailed = 4 ) // ErrMissingTerminalSnapshot means the Sweeper exited successfully without publishing a // terminal snapshot, which violates the producer protocol. -var ErrMissingTerminalSnapshot = errors.New("sweeper exited without publishing a terminal snapshot") +var ErrMissingTerminalSnapshot = fmt.Errorf("%w: sweeper exited without publishing a terminal snapshot", ErrProtocolViolation) // ErrRunFailed means the Sweeper published a terminal Failed snapshot. The snapshot is // reconciled first; the error keeps a producer that reports failure but exits 0 from @@ -73,8 +77,8 @@ func ExitCode(err error) int { switch { case err == nil: return ExitAcknowledged - case errors.Is(err, ErrMissingTerminalSnapshot): - return ExitMissingTerminalSnapshot + case errors.Is(err, ErrProtocolViolation): + return ExitProtocolViolation case errors.Is(err, ErrRunFailed): return ExitRunFailed default: @@ -121,17 +125,29 @@ type Publisher struct { RunName string PollInterval time.Duration - lastRaw []byte lastStatus *RunStatus haveSnapshot bool // a snapshot has been reconciled + round int32 + evaluated int32 terminal bool // the reconciled snapshot was terminal phase string } +// maxObjectName is the Kubernetes limit on an object name (DNS subdomain). +const maxObjectName = 253 + // CandidateName is the DGDC name for a candidate id. It depends only on the run and the -// stable id, never on rank. +// stable id, never on rank. A run name too long to leave room for the id is truncated +// and a hash of the full run name is added, so names stay unique and within the limit. func CandidateName(runName, id string) string { - return fmt.Sprintf("%s-%s", runName, id) + name := runName + "-" + id + if len(name) <= maxObjectName { + return name + } + sum := sha256.Sum256([]byte(runName)) + hash := hex.EncodeToString(sum[:6]) + room := maxObjectName - len(id) - len(hash) - 2 + return strings.TrimRight(runName[:room], "-.") + "-" + hash + "-" + id } // Reconcile applies one snapshot: create missing candidates, patch the run status @@ -227,8 +243,9 @@ func (p *Publisher) patchStatus(ctx context.Context, status RunStatus) error { return nil } -// syncOnce reads the snapshot file and reconciles it if it changed. A missing file or -// an unchanged snapshot is not an error. +// syncOnce reads the snapshot file and reconciles it. A missing file is not an error. +// An unchanged snapshot is reconciled again on purpose: the diff makes that free when +// the cluster matches, and repairs it (for example a deleted candidate) when it does not. func (p *Publisher) syncOnce(ctx context.Context) error { data, err := os.ReadFile(filepath.Join(p.SnapshotDir, SnapshotFileName)) if errors.Is(err, os.ErrNotExist) { @@ -237,25 +254,40 @@ func (p *Publisher) syncOnce(ctx context.Context) error { if err != nil { return err } - if p.lastRaw != nil && string(p.lastRaw) == string(data) { - return nil - } snap, err := ParseSnapshot(data) if err != nil { return err } + if err := p.checkProgress(snap); err != nil { + return err + } if err := p.Reconcile(ctx, snap); err != nil { return err } - // Record exactly the bytes that were reconciled, even if the file was replaced - // since: the next sync will see the newer content as a change. - p.lastRaw = data p.haveSnapshot = true + p.round = snap.Progress.Round + p.evaluated = snap.Progress.Evaluated p.terminal = snap.Run.Terminal p.phase = snap.Run.Phase return nil } +// checkProgress rejects a snapshot that moves progress backwards: an older round, or +// fewer evaluated points within the same round. A terminal snapshot is exempt from the +// evaluated check because its count is the search's final total, not the live counter. +func (p *Publisher) checkProgress(snap *Snapshot) error { + if !p.haveSnapshot { + return nil + } + if snap.Progress.Round < p.round { + return violation("snapshot round regressed from %d to %d", p.round, snap.Progress.Round) + } + if !snap.Run.Terminal && snap.Progress.Round == p.round && snap.Progress.Evaluated < p.evaluated { + return violation("snapshot evaluated count regressed from %d to %d in round %d", p.evaluated, snap.Progress.Evaluated, p.round) + } + return nil +} + // Run consumes snapshots until a terminal snapshot has been reconciled or the Sweeper // container has exited. See the package comment for the exit semantics. func (p *Publisher) Run(ctx context.Context) error { @@ -293,7 +325,6 @@ func (p *Publisher) Run(ctx context.Context) error { } } -// terminalResult is the outcome once a terminal snapshot has been reconciled. func (p *Publisher) terminalResult() error { if p.phase == PhaseFailed { return ErrRunFailed diff --git a/deploy/operator/internal/dgdrrunpublisher/publisher_test.go b/deploy/operator/internal/dgdrrunpublisher/publisher_test.go index a9e78cc90eac..e1d62c901c8c 100644 --- a/deploy/operator/internal/dgdrrunpublisher/publisher_test.go +++ b/deploy/operator/internal/dgdrrunpublisher/publisher_test.go @@ -40,6 +40,8 @@ type fakeCluster struct { creates []string deletes []string patches []RunStatus + ops []string // ordered log of every mutation + incomplete map[string]bool state SweeperState patchErr error createErr error @@ -47,7 +49,7 @@ type fakeCluster struct { } func newFakeCluster() *fakeCluster { - return &fakeCluster{candidates: map[string]string{}, created: map[string]dgdcreconcile.DesiredCandidate{}} + return &fakeCluster{candidates: map[string]string{}, created: map[string]dgdcreconcile.DesiredCandidate{}, incomplete: map[string]bool{}} } func (f *fakeCluster) ListCandidates(context.Context) ([]dgdcreconcile.CurrentDGDC, error) { @@ -55,7 +57,7 @@ func (f *fakeCluster) ListCandidates(context.Context) ([]dgdcreconcile.CurrentDG defer f.mu.Unlock() var out []dgdcreconcile.CurrentDGDC for name, id := range f.candidates { - out = append(out, dgdcreconcile.CurrentDGDC{Name: name, ID: id}) + out = append(out, dgdcreconcile.CurrentDGDC{Name: name, ID: id, Incomplete: f.incomplete[name]}) } return out, nil } @@ -67,6 +69,8 @@ func (f *fakeCluster) CreateCandidate(_ context.Context, name string, c dgdcreco if f.createErr != nil { return f.createErr } + delete(f.incomplete, name) + f.ops = append(f.ops, "create:"+name) f.candidates[name] = c.ID f.created[name] = c f.creates = append(f.creates, name) @@ -77,6 +81,7 @@ func (f *fakeCluster) DeleteCandidate(_ context.Context, name string) error { f.mu.Lock() defer f.mu.Unlock() delete(f.candidates, name) + f.ops = append(f.ops, "delete:"+name) f.deletes = append(f.deletes, name) return nil } @@ -87,6 +92,7 @@ func (f *fakeCluster) PatchRunStatus(_ context.Context, s RunStatus) error { if f.patchErr != nil { return f.patchErr } + f.ops = append(f.ops, "patch") f.patches = append(f.patches, s) return nil } @@ -103,6 +109,8 @@ func (f *fakeCluster) setState(s SweeperState) { f.state = s } +const validManifest = "kind: DynamoGraphDeployment\nspec: {}\n" + type cand struct { id string failed bool @@ -120,7 +128,7 @@ func snapshotJSON(t *testing.T, phase string, cands ...cand) []byte { entry["error"] = "boom" } else { entry["outcome"] = OutcomeMaterialized - entry["manifest"] = "kind: DynamoGraphDeployment\n" + entry["manifest"] = validManifest } list = append(list, entry) } @@ -189,7 +197,7 @@ func TestEvaluatedPointReachesTheClusterComplete(t *testing.T) { t.Fatal(err) } got := fc.created["run-a"] - if got.Spec != "kind: DynamoGraphDeployment\n" { + if got.Spec != validManifest { t.Fatalf("manifest = %q", got.Spec) } if v, ok := got.Parameters["tp"]; !ok || v != float64(2) { @@ -225,20 +233,6 @@ func TestReturningCandidateIsRecreatedIdentically(t *testing.T) { } } -// A published DGDC is immutable: the publisher's view of the cluster has no way to -// update one, so no code path can modify an evaluated point after its initial status. -func TestClusterInterfaceCannotUpdateCandidates(t *testing.T) { - typ := reflect.TypeOf((*Cluster)(nil)).Elem() - var methods []string - for i := 0; i < typ.NumMethod(); i++ { - methods = append(methods, typ.Method(i).Name) - } - want := []string{"CreateCandidate", "DeleteCandidate", "ListCandidates", "PatchRunStatus", "SweeperState"} - if !reflect.DeepEqual(methods, want) { - t.Fatalf("Cluster methods = %v, want %v: adding a candidate-mutating call breaks DGDC immutability", methods, want) - } -} - func TestRepeatedSnapshotCausesNoWrites(t *testing.T) { p, fc, _ := newPublisher(t) ctx := context.Background() @@ -264,6 +258,10 @@ func TestDroppedCandidateIsDeletedAfterStatusPatch(t *testing.T) { if got := fc.patches[len(fc.patches)-1].CandidateNames; !reflect.DeepEqual(got, []string{"run-a"}) { t.Fatalf("status refs = %v", got) } + want := []string{"create:run-a", "create:run-b", "patch", "patch", "delete:run-b"} + if !reflect.DeepEqual(fc.ops, want) { + t.Fatalf("ops = %v, want %v", fc.ops, want) + } } func TestFailedMaterializationIsNotCreatedButCounted(t *testing.T) { @@ -357,14 +355,19 @@ func TestParseSnapshotValidation(t *testing.T) { "schema": `{"schemaVersion":2,"run":{"phase":"Running","terminal":false}}`, "phase": `{"schemaVersion":1,"run":{"phase":"Weird","terminal":false}}`, "terminal flag": `{"schemaVersion":1,"run":{"phase":"Succeeded","terminal":false}}`, - "duplicate": `{"schemaVersion":1,"run":{"phase":"Running","terminal":false},"candidates":[{"id":"a","outcome":"materialized","manifest":"x"},{"id":"a","outcome":"materialized","manifest":"x"}]}`, + "duplicate": `{"schemaVersion":1,"run":{"phase":"Running","terminal":false},"candidates":[{"id":"a","outcome":"materialized","manifest":"kind: DynamoGraphDeployment\\nspec: {}\\n"},{"id":"a","outcome":"materialized","manifest":"kind: DynamoGraphDeployment\\nspec: {}\\n"}]}`, "no manifest": `{"schemaVersion":1,"run":{"phase":"Running","terminal":false},"candidates":[{"id":"a","outcome":"materialized"}]}`, "no error": `{"schemaVersion":1,"run":{"phase":"Running","terminal":false},"candidates":[{"id":"a","outcome":"materialization_failed"}]}`, - "empty id": `{"schemaVersion":1,"run":{"phase":"Running","terminal":false},"candidates":[{"id":"","outcome":"materialized","manifest":"x"}]}`, + "unsafe id": `{"schemaVersion":1,"run":{"phase":"Running","terminal":false},"candidates":[{"id":"Not_Valid","outcome":"materialized","manifest":"kind: DynamoGraphDeployment\\nspec: {}\\n"}]}`, + "long id": `{"schemaVersion":1,"run":{"phase":"Running","terminal":false},"candidates":[{"id":"%s","outcome":"materialized","manifest":"kind: DynamoGraphDeployment\\nspec: {}\\n"}]}`, + "wrong kind": `{"schemaVersion":1,"run":{"phase":"Running","terminal":false},"candidates":[{"id":"a","outcome":"materialized","manifest":"kind: ConfigMap\\nspec: {}\\n"}]}`, + "no spec": `{"schemaVersion":1,"run":{"phase":"Running","terminal":false},"candidates":[{"id":"a","outcome":"materialized","manifest":"kind: DynamoGraphDeployment\\n"}]}`, + "empty id": `{"schemaVersion":1,"run":{"phase":"Running","terminal":false},"candidates":[{"id":"","outcome":"materialized","manifest":"kind: DynamoGraphDeployment\\nspec: {}\\n"}]}`, } + bad["long id"] = strings.Replace(bad["long id"], "%s", strings.Repeat("a", 64), 1) for name, doc := range bad { - if _, err := ParseSnapshot([]byte(doc)); err == nil { - t.Errorf("%s: want error", name) + if _, err := ParseSnapshot([]byte(doc)); !errors.Is(err, ErrProtocolViolation) { + t.Errorf("%s: want protocol violation, got %v", name, err) } } } @@ -398,8 +401,6 @@ func TestRunTerminalSnapshotAcknowledges(t *testing.T) { } } -// A terminal Failed snapshot has its projection reconciled, then fails the publisher -// whether or not the Sweeper itself exited non-zero. func TestRunTerminalFailedSnapshotIsReconciledThenFails(t *testing.T) { for name, state := range map[string]SweeperState{ "sweeper exit 0": {Exited: true, ExitCode: 0}, @@ -421,21 +422,6 @@ func TestRunTerminalFailedSnapshotIsReconciledThenFails(t *testing.T) { } } -func TestRunLabelValueIsLabelSafeAndStable(t *testing.T) { - if got := RunLabelValue("short-run"); got != "short-run" { - t.Fatalf("short names are kept: %q", got) - } - long := strings.Repeat("a", 100) + "-x" - other := strings.Repeat("a", 100) + "-y" - got := RunLabelValue(long) - if len(got) > 63 || got == RunLabelValue(other) || got != RunLabelValue(long) { - t.Fatalf("label = %q (%d)", got, len(got)) - } - if last := got[len(got)-1]; !(last >= '0' && last <= '9' || last >= 'a' && last <= 'f') { - t.Fatalf("must end alphanumeric: %q", got) - } -} - func TestRunReconcilesTerminalSnapshotWrittenAfterRunningOne(t *testing.T) { p, fc, dir := newPublisher(t) writeSnapshot(t, dir, snapshotJSON(t, PhaseRunning, cand{id: "a"})) @@ -484,7 +470,7 @@ func TestRunCleanExitWithoutTerminalSnapshotIsProtocolViolation(t *testing.T) { writeSnapshot(t, dir, snapshotJSON(t, PhaseRunning, cand{id: "a"})) fc.setState(SweeperState{Exited: true, ExitCode: 0}) err := runWithTimeout(t, p) - if !errors.Is(err, ErrMissingTerminalSnapshot) || ExitCode(err) != ExitMissingTerminalSnapshot { + if !errors.Is(err, ErrMissingTerminalSnapshot) || ExitCode(err) != ExitProtocolViolation { t.Fatalf("err = %v", err) } if !reflect.DeepEqual(fc.creates, []string{"run-a"}) { @@ -552,3 +538,105 @@ func TestRunContextCancelledReturnsError(t *testing.T) { t.Fatalf("err = %v", err) } } + +func TestCandidateNameIsBoundedAndUnique(t *testing.T) { + if got := CandidateName("run", "a"); got != "run-a" { + t.Fatalf("short names are unchanged: %q", got) + } + id := "evaluated-point-0123456789ab" + long := strings.Repeat("r", 253) + other := strings.Repeat("r", 252) + "s" + name := CandidateName(long, id) + if len(name) > 253 || !strings.HasSuffix(name, "-"+id) || name == CandidateName(other, id) || name != CandidateName(long, id) { + t.Fatalf("name = %q (%d)", name, len(name)) + } +} + +func TestUnchangedSnapshotRepairsDrift(t *testing.T) { + p, fc, dir := newPublisher(t) + ctx := context.Background() + writeSnapshot(t, dir, snapshotJSON(t, PhaseRunning, cand{id: "a"})) + if err := p.syncOnce(ctx); err != nil { + t.Fatal(err) + } + delete(fc.candidates, "run-a") // someone deletes the referenced candidate + if err := p.syncOnce(ctx); err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(fc.creates, []string{"run-a", "run-a"}) { + t.Fatalf("creates = %v", fc.creates) + } +} + +func TestCandidateWithoutStatusIsCompletedBeforeItIsReferenced(t *testing.T) { + p, fc, dir := newPublisher(t) + ctx := context.Background() + fc.candidates["run-a"] = "a" + fc.incomplete["run-a"] = true // created, but the status write failed + writeSnapshot(t, dir, snapshotJSON(t, PhaseRunning, cand{id: "a"})) + if err := p.syncOnce(ctx); err != nil { + t.Fatal(err) + } + if want := []string{"create:run-a", "patch"}; !reflect.DeepEqual(fc.ops, want) { + t.Fatalf("ops = %v, want %v", fc.ops, want) + } +} + +func writeProgress(t *testing.T, dir, phase string, round, evaluated int) { + t.Helper() + doc := map[string]any{} + if err := json.Unmarshal(snapshotJSON(t, phase, cand{id: "a"}), &doc); err != nil { + t.Fatal(err) + } + doc["progress"] = map[string]any{"round": round, "evaluated": evaluated} + data, err := json.Marshal(doc) + if err != nil { + t.Fatal(err) + } + writeSnapshot(t, dir, data) +} + +func TestRegressingSnapshotIsRejectedBeforeReconcile(t *testing.T) { + for name, regressed := range map[string][2]int{"older round": {2, 99}, "fewer evaluated in the same round": {3, 4}} { + t.Run(name, func(t *testing.T) { + p, fc, dir := newPublisher(t) + ctx := context.Background() + writeProgress(t, dir, PhaseRunning, 3, 5) + if err := p.syncOnce(ctx); err != nil { + t.Fatal(err) + } + ops := len(fc.ops) + writeProgress(t, dir, PhaseRunning, regressed[0], regressed[1]) + err := p.syncOnce(ctx) + if !errors.Is(err, ErrProtocolViolation) || ExitCode(err) != ExitProtocolViolation { + t.Fatalf("err = %v", err) + } + if len(fc.ops) != ops { + t.Fatalf("a regressing snapshot must not be reconciled: %v", fc.ops[ops:]) + } + }) + } +} + +func TestTerminalSnapshotMayReportItsFinalEvaluatedTotal(t *testing.T) { + p, _, dir := newPublisher(t) + ctx := context.Background() + writeProgress(t, dir, PhaseRunning, 3, 9) + if err := p.syncOnce(ctx); err != nil { + t.Fatal(err) + } + writeProgress(t, dir, PhaseSucceeded, 3, 7) + if err := p.syncOnce(ctx); err != nil { + t.Fatalf("terminal snapshot rejected: %v", err) + } +} + +func TestInvalidFinalSnapshotIsAProtocolViolationExit(t *testing.T) { + p, fc, dir := newPublisher(t) + writeSnapshot(t, dir, []byte(`{"schemaVersion":2}`)) + fc.setState(SweeperState{Exited: true, ExitCode: 0}) + err := runWithTimeout(t, p) + if ExitCode(err) != ExitProtocolViolation { + t.Fatalf("err = %v, exit = %d", err, ExitCode(err)) + } +} diff --git a/deploy/operator/internal/dgdrrunpublisher/snapshot.go b/deploy/operator/internal/dgdrrunpublisher/snapshot.go index daf3e14b545e..ef5ad6f2b9ce 100644 --- a/deploy/operator/internal/dgdrrunpublisher/snapshot.go +++ b/deploy/operator/internal/dgdrrunpublisher/snapshot.go @@ -18,11 +18,44 @@ package dgdrrunpublisher import ( + "errors" "fmt" + "regexp" "sigs.k8s.io/yaml" ) +// ErrProtocolViolation means the Sweeper broke the snapshot contract: a snapshot that +// cannot be decoded or validated, one that moves progress backwards, or a missing +// terminal snapshot. It maps to ExitProtocolViolation. +var ErrProtocolViolation = errors.New("snapshot protocol violation") + +func violation(format string, args ...any) error { + return fmt.Errorf("%w: %s", ErrProtocolViolation, fmt.Sprintf(format, args...)) +} + +// maxCandidateIDLength keeps ids well inside the 63-character Kubernetes label-value +// limit, since the id is recorded verbatim as a label on the candidate. +const maxCandidateIDLength = 48 + +// candidateIDPattern is a DNS-1123 label, valid both as a label value and inside an +// object name. +var candidateIDPattern = regexp.MustCompile(`^[a-z0-9]([-a-z0-9]*[a-z0-9])?$`) + +func validateManifest(id, manifest string) error { + var doc map[string]any + if err := yaml.Unmarshal([]byte(manifest), &doc); err != nil { + return violation("manifest of candidate %q is not a YAML mapping: %v", id, err) + } + if doc["kind"] != "DynamoGraphDeployment" { + return violation("manifest of candidate %q is not a DynamoGraphDeployment", id) + } + if _, ok := doc["spec"].(map[string]any); !ok { + return violation("manifest of candidate %q has no spec", id) + } + return nil +} + // SnapshotSchemaVersion is the only wire version this publisher understands. const SnapshotSchemaVersion = 1 @@ -76,39 +109,45 @@ type SnapshotCandidate struct { func ParseSnapshot(data []byte) (*Snapshot, error) { var snap Snapshot if err := yaml.Unmarshal(data, &snap); err != nil { - return nil, fmt.Errorf("decoding snapshot: %w", err) + return nil, violation("decoding snapshot: %v", err) } if snap.SchemaVersion != SnapshotSchemaVersion { - return nil, fmt.Errorf("unsupported snapshot schemaVersion %d (want %d)", snap.SchemaVersion, SnapshotSchemaVersion) + return nil, violation("unsupported snapshot schemaVersion %d (want %d)", snap.SchemaVersion, SnapshotSchemaVersion) } switch snap.Run.Phase { case PhaseRunning, PhaseSucceeded, PhaseFailed: default: - return nil, fmt.Errorf("unknown run phase %q", snap.Run.Phase) + return nil, violation("unknown run phase %q", snap.Run.Phase) } if snap.Run.Terminal != (snap.Run.Phase != PhaseRunning) { - return nil, fmt.Errorf("run phase %q inconsistent with terminal=%t", snap.Run.Phase, snap.Run.Terminal) + return nil, violation("run phase %q inconsistent with terminal=%t", snap.Run.Phase, snap.Run.Terminal) } seen := make(map[string]struct{}, len(snap.Candidates)) for i, candidate := range snap.Candidates { if candidate.ID == "" { - return nil, fmt.Errorf("candidate %d has an empty id", i) + return nil, violation("candidate %d has an empty id", i) + } + if len(candidate.ID) > maxCandidateIDLength || !candidateIDPattern.MatchString(candidate.ID) { + return nil, violation("candidate id %q is not a lowercase DNS label of at most %d characters", candidate.ID, maxCandidateIDLength) } if _, dup := seen[candidate.ID]; dup { - return nil, fmt.Errorf("duplicate candidate id %q", candidate.ID) + return nil, violation("duplicate candidate id %q", candidate.ID) } seen[candidate.ID] = struct{}{} switch candidate.Outcome { case OutcomeMaterialized: if candidate.Manifest == "" { - return nil, fmt.Errorf("materialized candidate %q has no manifest", candidate.ID) + return nil, violation("materialized candidate %q has no manifest", candidate.ID) + } + if err := validateManifest(candidate.ID, candidate.Manifest); err != nil { + return nil, err } case OutcomeMaterializationFailed: if candidate.Error == "" { - return nil, fmt.Errorf("failed candidate %q has no error", candidate.ID) + return nil, violation("failed candidate %q has no error", candidate.ID) } default: - return nil, fmt.Errorf("candidate %q has unknown outcome %q", candidate.ID, candidate.Outcome) + return nil, violation("candidate %q has unknown outcome %q", candidate.ID, candidate.Outcome) } } return &snap, nil From ee711d27861de098c184c525551df1c7a0593ab8 Mon Sep 17 00:00:00 2001 From: devivasudevan <49675305+devivasudevan@users.noreply.github.com> Date: Wed, 7 Oct 2026 13:18:56 +0200 Subject: [PATCH 04/40] feat(aisimulate): add dgdr_run live output adapter Writes an atomic pod-local snapshot for the DGDR run publisher: stable rank-independent candidate ids, bounded and coalesced updates, and a terminal snapshot as the completion barrier. Candidate-to-DGD rendering reuses dgd.renderers. Live updates need aisimulate#384 (subscribe(config, context)); without it the adapter writes a single snapshot from write(). Routes output/dgdr_run/ to the planner and operator owners, like output/dgd/. --- .github/codeowners/areas.yaml | 4 + CODEOWNERS | 1 + .../aisimulate/output/dgdr_run/__init__.py | 34 + .../aisimulate/output/dgdr_run/adapter.py | 602 ++++++++++++ .../aisimulate/output/dgdr_run/snapshot.py | 148 +++ .../aisimulate/tests/dgdr_run/test_adapter.py | 884 ++++++++++++++++++ .../tests/dgdr_run/test_snapshot.py | 154 +++ pyproject.toml | 1 + 8 files changed, 1828 insertions(+) create mode 100644 components/src/dynamo/aisimulate/output/dgdr_run/__init__.py create mode 100644 components/src/dynamo/aisimulate/output/dgdr_run/adapter.py create mode 100644 components/src/dynamo/aisimulate/output/dgdr_run/snapshot.py create mode 100644 components/src/dynamo/aisimulate/tests/dgdr_run/test_adapter.py create mode 100644 components/src/dynamo/aisimulate/tests/dgdr_run/test_snapshot.py diff --git a/.github/codeowners/areas.yaml b/.github/codeowners/areas.yaml index 369eb3a60879..2f125e283ee4 100644 --- a/.github/codeowners/areas.yaml +++ b/.github/codeowners/areas.yaml @@ -874,6 +874,10 @@ shared: owners: - planner - operator +- glob: components/src/dynamo/aisimulate/output/dgdr_run/ + owners: + - planner + - operator - glob: components/src/dynamo/planner/connectors/kubernetes.py owners: - planner diff --git a/CODEOWNERS b/CODEOWNERS index 15843b08b972..c3134d6d3f21 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -749,6 +749,7 @@ requirements.txt /deploy/operator/internal/webhook/**/*checkpoint* @ai-dynamo/dynamo-operator-codeowners @ai-dynamo/dynamo-gms-codeowners /.github/actions/build-deploy-component/action.yml @ai-dynamo/dynamo-ops-codeowners @ai-dynamo/dynamo-operator-codeowners @ai-dynamo/dynamo-gms-codeowners @ai-dynamo/dynamo-planner-codeowners /.github/actions/check-deploy-component/action.yml @ai-dynamo/dynamo-ops-codeowners @ai-dynamo/dynamo-operator-codeowners @ai-dynamo/dynamo-gms-codeowners +/components/src/dynamo/aisimulate/output/dgdr_run/ @ai-dynamo/dynamo-planner-codeowners @ai-dynamo/dynamo-operator-codeowners /components/src/dynamo/common/backend/publisher.py @ai-dynamo/dynamo-runtime-codeowners @ai-dynamo/dynamo-kv-memory-codeowners /components/src/dynamo/common/backend/telemetry.py @ai-dynamo/dynamo-runtime-codeowners @ai-dynamo/dynamo-observability-codeowners /components/src/dynamo/common/utils/image_utils.py @ai-dynamo/dynamo-runtime-codeowners @ai-dynamo/dynamo-diffusion-codeowners diff --git a/components/src/dynamo/aisimulate/output/dgdr_run/__init__.py b/components/src/dynamo/aisimulate/output/dgdr_run/__init__.py new file mode 100644 index 000000000000..290ce406d8f5 --- /dev/null +++ b/components/src/dynamo/aisimulate/output/dgdr_run/__init__.py @@ -0,0 +1,34 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Live DynamoGraphDeploymentRun output adapter for AISimulate (``dgdr_run``).""" + +from dynamo.aisimulate.output.dgdr_run.adapter import ( + DGDRRunOutputAdapter, + DGDRRunOutputConfig, + DGDRRunOutputPlugin, + candidate_id_for, + create_adapter, +) +from dynamo.aisimulate.output.dgdr_run.snapshot import ( + SNAPSHOT_FILE_NAME, + CandidateOutcome, + DGDRRunSnapshot, + RunPhase, + SnapshotCandidate, + write_snapshot, +) + +__all__ = [ + "SNAPSHOT_FILE_NAME", + "CandidateOutcome", + "DGDRRunOutputAdapter", + "DGDRRunOutputConfig", + "DGDRRunOutputPlugin", + "DGDRRunSnapshot", + "RunPhase", + "SnapshotCandidate", + "candidate_id_for", + "create_adapter", + "write_snapshot", +] diff --git a/components/src/dynamo/aisimulate/output/dgdr_run/adapter.py b/components/src/dynamo/aisimulate/output/dgdr_run/adapter.py new file mode 100644 index 000000000000..e8739203fd53 --- /dev/null +++ b/components/src/dynamo/aisimulate/output/dgdr_run/adapter.py @@ -0,0 +1,602 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Live DGDRRun output adapter (``--output dgdr_run``). + +Independent of the standalone ``dgd`` adapter, which is called once with the +final recommendation. This adapter subscribes to Sweeper's ``on_round`` and +``on_candidate`` callbacks for the whole search, keeps the *current bounded +projection* of the run in memory, and periodically publishes it as an +atomically replaced ``dgdr_run_snapshot.yaml`` (see ``snapshot.py``) that the +publisher sidecar in the same pod consumes. + +Both adapters share one Candidate-to-DGD implementation through +``dynamo.aisimulate.output.dgd.renderers`` (its public +CandidateMaterializationError / DGDGenerationOptions / render_dgd surface); +nothing else is shared. + +Design points (all adapter-specific, none of them shared with ``dgd``): + +* **Callbacks never block and never lose state.** They only update the latest + in-memory projection under a short lock and wake the writer. Nothing is + queued, so nothing can be dropped under pressure: only *file writes* are + coalesced, never logical state changes. +* **Bounded.** At most ``max_candidates`` points are retained and published, + no matter how many are evaluated. The bound comes from the run's + ``recommendation.maxCandidates``; it is not derived from any transport. +* **Rate limited.** The writer publishes the latest complete state at most once + per ``snapshot_interval_seconds``. The initial snapshot (``start()``) and the + terminal snapshot (``close()``) bypass the limit. +* **Materialize once.** A point's DGD is rendered the first time it enters the + projection and cached by its stable id, so progress or ordering changes never + re-render retained points. +* **Terminal snapshot is a completion barrier.** ``close()`` stops accepting + updates, folds in everything accepted so far, and synchronously writes the + terminal snapshot before returning. A failure to write it is raised, never + swallowed, so the Sweeper container cannot exit successfully without it. + +Scalar searches only for now (best-first ordering by score); a Pareto +selection policy is a separate piece of work and is rejected explicitly. +""" + +from __future__ import annotations + +import hashlib +import json +import logging +import threading +import time +from collections.abc import Mapping, Sequence +from dataclasses import dataclass, fields +from pathlib import Path +from typing import Any, Literal, Protocol + +import yaml +from aisimulate.output_adapter import ( + OUTPUT_ADAPTER_API_VERSION, + RecommendationOutputCallbacks, +) +from aisimulate.sweeper.config import SmartSearchConfig +from aisimulate.sweeper.result import SweepResult +from pydantic import BaseModel, ConfigDict, Field + +from dynamo.aisimulate.output.dgd.renderers import ( + CandidateMaterializationError, + DGDGenerationOptions, + render_dgd, +) +from dynamo.aisimulate.output.dgdr_run.snapshot import ( + SNAPSHOT_FILE_NAME, + CandidateOutcome, + DGDRRunSnapshot, + RunPhase, + SnapshotCandidate, + write_snapshot, +) + +logger = logging.getLogger(__name__) + +_ID_PREFIX = "evaluated-point-" +_MAX_RENDER_ATTEMPTS = 3 +_MAX_ERROR_MESSAGE = 500 +_ID_HEX_LENGTH = 12 + + +class CandidateRecordLike(Protocol): + """The subset of aisimulate's ``CandidateRecord`` this adapter reads.""" + + candidate_id: str + status: Any # aisimulate.sweeper.result.CandidateStatus + score: float | None + used_gpus: int | None + config: dict[str, Any] + + +def _plain(value: Any) -> Any: + """Copy ``value`` into plain, YAML-safe containers (detached from the caller).""" + return json.loads(json.dumps(value, default=str, sort_keys=True)) + + +def candidate_id_for(parameters: Mapping[str, Any]) -> str: + """Stable identity of an evaluated point. + + Depends only on the point's resolved parameters: never on rank, list + position, or the order in which points were observed. + """ + canonical = json.dumps(parameters, sort_keys=True, separators=(",", ":")) + digest = hashlib.sha256(canonical.encode("utf-8")).hexdigest() + return f"{_ID_PREFIX}{digest[:_ID_HEX_LENGTH]}" + + +def _is_feasible(record: CandidateRecordLike) -> bool: + # By value, not identity: callers may pass aisimulate's CandidateStatus + # enum, a plain string, or a test double. + return str(getattr(record.status, "value", record.status)) == "feasible" + + +@dataclass(frozen=True) +class _Retained: + """One retained point: immutable evaluation facts, no rendering.""" + + id: str + score: float + used_gpus: int + parameters: dict[str, Any] + metrics: dict[str, Any] + + @property + def rank_key(self) -> tuple[float, int, str]: + """Ascending sort key: best first. + + Highest score, then fewer GPUs, then id so that equal points order + deterministically. Matches Sweeper's own scalar ranking. + """ + return (-self.score, self.used_gpus, self.id) + + +@dataclass(frozen=True) +class _Terminal: + phase: RunPhase + message: str + error: str + + +class DGDRRunOutputConfig(BaseModel): + """Configuration accepted for ``--output dgdr_run``.""" + + model_config = ConfigDict(extra="forbid") + + name: str = Field(min_length=1, pattern=r"^[a-z0-9]([-a-z0-9]*[a-z0-9])?$") + snapshot_dir: str | None = Field(default=None, min_length=1) + namespace: str | None = None + generator: Literal["aic", "direct"] = "aic" + runtime_image: str = Field(min_length=1) + runtime_version_override: str | None = None + num_gpus_per_node: int = Field(gt=0) + max_candidates: int = Field(default=5, ge=1) + snapshot_interval_seconds: float = Field(default=5.0, ge=0) + + def generation_options(self) -> DGDGenerationOptions: + return DGDGenerationOptions( + runtime_image=self.runtime_image, + runtime_version_override=self.runtime_version_override, + namespace=self.namespace, + num_gpus_per_node=self.num_gpus_per_node, + ) + + +class DGDRRunOutputAdapter: + """Publishes the current bounded projection of one Sweeper run. + + One instance per run. Pass the bound methods to ``Sweeper.run`` and use the + adapter as a context manager (or call ``start()``/``close()``) so the + terminal snapshot is always written, including when the run raises:: + + with DGDRRunOutputAdapter(config, workload, snapshot_dir=path) as adapter: + sweeper.run(on_round=adapter.on_round, on_candidate=adapter.on_candidate) + """ + + def __init__( + self, + config: DGDRRunOutputConfig, + workload: Any, + *, + snapshot_dir: Path | None = None, + is_pareto: bool = False, + ) -> None: + if is_pareto: + raise NotImplementedError( + "dgdr_run only supports scalar objectives so far; Pareto " + "retention needs its own selection policy rather than an " + "approximation of top-N-by-score" + ) + directory = snapshot_dir if snapshot_dir is not None else config.snapshot_dir + if directory is None: + raise ValueError("a snapshot directory is required") + self._directory = Path(directory) + self._config = config + self._options = config.generation_options() + self._workload = workload + self._max_candidates = config.max_candidates + self._interval = config.snapshot_interval_seconds + + # Latest in-memory desired state. Callbacks mutate it under _lock. + self._lock = threading.Lock() + self._retained: dict[str, _Retained] = {} + self._round_no = 0 + self._evaluated = 0 + self._terminal: _Terminal | None = None + self._accepting = True + + # Writer coordination. + self._wake = threading.Event() + self._stop = threading.Event() + self._thread: threading.Thread | None = None + self._next_allowed = float("-inf") + + # Materialization cache, keyed by stable id. Touched only while + # holding _publish_lock. A failure is retried on later writes up to + # _MAX_RENDER_ATTEMPTS in total, because a first render can fail + # transiently (cold start); after that it is final. + self._publish_lock = threading.Lock() + self._materialized: dict[str, SnapshotCandidate] = {} + self._render_attempts: dict[str, int] = {} + self._writes = 0 + + # -- observability --------------------------------------------------- + + @property + def snapshot_writes(self) -> int: + """Number of snapshots written so far (initial and terminal included).""" + return self._writes + + @property + def snapshot_path(self) -> Path: + return self._directory / SNAPSHOT_FILE_NAME + + # -- Sweeper callbacks: update latest state, never block, never raise -- + + def on_candidate(self, record: CandidateRecordLike) -> None: + try: + retained = self._retain(record) if _is_feasible(record) else None + except Exception: # a callback must not break the search + logger.exception("dgdr_run: ignoring candidate that could not be read") + retained = None + with self._lock: + if not self._accepting: + return + self._evaluated += 1 + # First observation wins: an evaluated point is immutable, so a + # re-evaluation of the same point never changes what was published. + if retained is not None and retained.id not in self._retained: + self._retained[retained.id] = retained + if len(self._retained) > self._max_candidates: + worst = max(self._retained.values(), key=lambda r: r.rank_key) + del self._retained[worst.id] + self._wake.set() + + def on_round(self, round_number: int, candidates: list[Any]) -> None: + del candidates # cumulative feasible list: the adapter tracks its own set + with self._lock: + if not self._accepting: + return + self._round_no = max(self._round_no, round_number) + self._wake.set() + + def report_progress(self, *, evaluated: int, round_no: int = 0) -> None: + """Raise the evaluated count and round to authoritative values (never lowers them).""" + with self._lock: + if not self._accepting: + return + self._evaluated = max(self._evaluated, evaluated) + self._round_no = max(self._round_no, round_no) + self._wake.set() + + @staticmethod + def _retain(record: CandidateRecordLike) -> _Retained | None: + if record.score is None or record.used_gpus is None: + logger.warning( + "dgdr_run: feasible candidate %s has no score/used_gpus; skipped", + record.candidate_id, + ) + return None + parameters = _plain(dict(record.config)) + metrics: dict[str, Any] = { + "score": float(record.score), + "usedGpus": int(record.used_gpus), + } + objectives = getattr(record, "objectives", None) + if objectives: + metrics["objectives"] = _plain(dict(objectives)) + return _Retained( + id=candidate_id_for(parameters), + score=float(record.score), + used_gpus=int(record.used_gpus), + parameters=parameters, + metrics=metrics, + ) + + # -- lifecycle --------------------------------------------------------- + + def start(self) -> None: + """Write the initial snapshot immediately, then start the writer.""" + if self._thread is not None: + raise RuntimeError("DGDRRunOutputAdapter already started") + self._publish() + self._next_allowed = time.monotonic() + self._interval + self._thread = threading.Thread( + target=self._writer_loop, name="dgdr-run-snapshot-writer", daemon=True + ) + self._thread.start() + + def close( + self, + *, + phase: RunPhase = RunPhase.SUCCEEDED, + message: str = "", + error: str = "", + ) -> None: + """Write the terminal snapshot; returns only once it is on disk. + + Stops accepting updates, folds in every update accepted before this + call, bypasses the rate limit, and replaces the file synchronously. + Raises if the terminal snapshot cannot be written. Safe to call without + ``start()`` and safe to call twice (the second call is a no-op). + """ + if not phase.terminal: + raise ValueError("close() requires a terminal phase") + with self._lock: + if self._terminal is not None: + return + self._accepting = False + self._terminal = _Terminal(phase=phase, message=message, error=error) + self._stop.set() + self._wake.set() + if self._thread is not None: + self._thread.join() + self._publish() + + def finish_search(self) -> None: + """The search ended normally: flush the last live state and stop the writer. + + Not a terminal state by itself: the terminal ``Succeeded`` snapshot is written + afterwards by ``write()``, which sees the final result. A search that found no + feasible candidate never reaches ``write()``, so that case is recorded here as + ``Failed`` instead of being left looking in progress. + """ + with self._lock: + if self._terminal is not None: + return + nothing_found = not self._retained + if nothing_found: + self.close(phase=RunPhase.FAILED, message="no feasible candidate found") + return + self._stop.set() + self._wake.set() + if self._thread is not None: + self._thread.join() + self._publish() + + def fail(self, error: BaseException) -> None: + """The search raised: write the terminal ``Failed`` snapshot.""" + self.close( + phase=RunPhase.FAILED, + error=type(error).__name__, + message=str(error)[:_MAX_ERROR_MESSAGE], + ) + + def __enter__(self) -> DGDRRunOutputAdapter: + self.start() + return self + + def __exit__(self, exc_type: type[BaseException] | None, *_: Any) -> None: + if exc_type is None: + self.close(phase=RunPhase.SUCCEEDED) + return + try: + self.close(phase=RunPhase.FAILED, error=exc_type.__name__) + except Exception: # never mask the run's own exception + logger.exception("dgdr_run: could not write the terminal snapshot") + + # -- writer ---------------------------------------------------------- + + def _writer_loop(self) -> None: + while True: + self._wake.wait() + if self._stop.is_set(): + return + delay = self._next_allowed - time.monotonic() + if delay > 0 and self._stop.wait(delay): + return + try: + self._publish() + except Exception: # keep the writer alive; retry + logger.exception("dgdr_run: snapshot write failed; will retry") + self._wake.set() + self._next_allowed = time.monotonic() + self._interval + + def _publish(self) -> None: + with self._publish_lock: + with self._lock: + # Cleared together with the copy so no update is ever lost: an + # update after this point sets the event again. + self._wake.clear() + ordered = sorted(self._retained.values(), key=lambda r: r.rank_key) + round_no = self._round_no + evaluated = self._evaluated + terminal = self._terminal + candidates = tuple(self._materialize(item) for item in ordered) + live_ids = {item.id for item in ordered} + for stale in set(self._materialized) - live_ids: + del self._materialized[stale] + self._render_attempts.pop(stale, None) + write_snapshot( + self._directory, + DGDRRunSnapshot( + phase=terminal.phase if terminal else RunPhase.RUNNING, + round_no=round_no, + evaluated=evaluated, + candidates=candidates, + message=terminal.message if terminal else "", + error=terminal.error if terminal else "", + ), + ) + self._writes += 1 + + def _materialize(self, item: _Retained) -> SnapshotCandidate: + cached = self._materialized.get(item.id) + if cached is not None and ( + cached.outcome is CandidateOutcome.MATERIALIZED + or self._render_attempts.get(item.id, 0) >= _MAX_RENDER_ATTEMPTS + ): + return cached + self._render_attempts[item.id] = self._render_attempts.get(item.id, 0) + 1 + try: + manifest = render_dgd( + _RenderInput(config=item.parameters), + self._workload, + self._options, + dgd_name=f"{self._config.name}-{item.id.removeprefix(_ID_PREFIX)}", + renderer=self._config.generator, + ) + entry = SnapshotCandidate( + id=item.id, + outcome=CandidateOutcome.MATERIALIZED, + parameters=item.parameters, + metrics=item.metrics, + manifest=manifest, + ) + except CandidateMaterializationError as exc: + # Retried on later writes until _MAX_RENDER_ATTEMPTS, then final. + entry = SnapshotCandidate( + id=item.id, + outcome=CandidateOutcome.MATERIALIZATION_FAILED, + parameters=item.parameters, + metrics=item.metrics, + error=str(exc), + ) + self._materialized[item.id] = entry + return entry + + +@dataclass(frozen=True) +class _RenderInput: + """What ``render_dgd`` reads from a candidate: its resolved config.""" + + config: dict[str, Any] + + +class DGDRRunOutputPlugin: + """The object AISimulate discovers for ``--output dgdr_run``. + + AISimulate calls ``subscribe(config, context=...)`` once, in the process that + runs the search, and wires the returned callbacks into the Sweeper: that is + the live path (initial snapshot immediately, then coalesced snapshots). + ``write()`` runs afterwards in the CLI process, only when the search + succeeded with a selection, and publishes the authoritative terminal + snapshot from the final result. A search that fails or finds nothing never + reaches ``write()``, so it leaves no terminal snapshot; the publisher + detects that from the Sweeper container's exit. + """ + + name = "dgdr_run" + api_version = OUTPUT_ADAPTER_API_VERSION + + def subscribe( + self, config: Mapping[str, Any], context: Any = None + ) -> RecommendationOutputCallbacks | None: + """Validate the configuration and, when possible, start live publishing. + + ``context`` (``RecommendationOutputContext``) carries the Sweeper workload, + which rendering needs. Without a context or without ``snapshot_dir`` there + is nothing to publish live, so only the config is validated and the final + snapshot is still produced by ``write()``. + """ + resolved = DGDRRunOutputConfig.model_validate(dict(config)) + resolved.generation_options() + workload = getattr(context, "workload", None) + if workload is None or resolved.snapshot_dir is None: + return None + adapter = DGDRRunOutputAdapter(resolved, workload) + adapter.start() + lifecycle = {"on_complete": adapter.finish_search, "on_failure": adapter.fail} + # Older AISimulate releases have no lifecycle callbacks; the terminal snapshot + # then comes from write() alone and a failed run is detected by the publisher. + supported = {field.name for field in fields(RecommendationOutputCallbacks)} + return RecommendationOutputCallbacks( + on_candidate=adapter.on_candidate, + on_round=adapter.on_round, + **{name: hook for name, hook in lifecycle.items() if name in supported}, + ) + + def live( + self, + config: Mapping[str, Any], + *, + workload: Any, + is_pareto: bool = False, + ) -> DGDRRunOutputAdapter: + resolved = DGDRRunOutputConfig.model_validate(dict(config)) + return DGDRRunOutputAdapter(resolved, workload, is_pareto=is_pareto) + + def write( + self, + config: Mapping[str, Any], + *, + result: SweepResult, + output_dir: Path, + ) -> Sequence[str | Path]: + resolved = DGDRRunOutputConfig.model_validate(dict(config)) + if result.views.pareto_front: + raise NotImplementedError("dgdr_run does not support Pareto results yet") + selected = list(result.selected_candidates)[: resolved.max_candidates] + if not selected: + raise CandidateMaterializationError("no feasible candidate found") + workload = SmartSearchConfig.model_validate(result.provenance.config).workload + directory = Path(resolved.snapshot_dir or output_dir) + adapter = DGDRRunOutputAdapter(resolved, workload, snapshot_dir=directory) + for candidate in selected: + adapter.on_candidate(_as_feasible(candidate)) + counts = getattr(result, "counts", None) + evaluated = getattr(counts, "evaluated", None) + adapter.report_progress( + evaluated=evaluated if isinstance(evaluated, int) else 0, + # The final result does not carry the round, but the live snapshot + # written by the search process does; progress never goes backwards. + round_no=_last_published_round(directory), + ) + adapter.close(phase=RunPhase.SUCCEEDED) + # AISimulate requires reported artifacts to live under output_dir; a snapshot + # on the shared volume does not, and is consumed by the publisher instead. + if directory.resolve() == Path(output_dir).resolve(): + return [Path(SNAPSHOT_FILE_NAME)] + return [] + + +def _last_published_round(directory: Path) -> int: + """The round recorded by the live snapshot already in ``directory``, else 0.""" + try: + loaded = yaml.safe_load((directory / SNAPSHOT_FILE_NAME).read_text()) + round_no = loaded["progress"]["round"] + except (OSError, yaml.YAMLError, KeyError, TypeError): + return 0 + return round_no if isinstance(round_no, int) and round_no > 0 else 0 + + +@dataclass(frozen=True) +class _FinalCandidate: + candidate_id: str + config: dict[str, Any] + score: float | None + used_gpus: int | None + status: str = "feasible" + + +def _as_feasible(candidate: Any) -> _FinalCandidate: + score = getattr(candidate, "score", None) + used_gpus = getattr(candidate, "used_gpus", None) + if score is None or used_gpus is None: + raise CandidateMaterializationError( + "selected candidate has no score/used_gpus, so it cannot be ranked" + ) + return _FinalCandidate( + candidate_id=str(getattr(candidate, "candidate_id", "")), + config=dict(candidate.config), + score=score, + used_gpus=used_gpus, + ) + + +def create_adapter() -> DGDRRunOutputPlugin: + """Create the Dynamo plugin discovered by AISimulate.""" + return DGDRRunOutputPlugin() + + +__all__ = [ + "CandidateRecordLike", + "DGDRRunOutputAdapter", + "DGDRRunOutputConfig", + "DGDRRunOutputPlugin", + "candidate_id_for", + "create_adapter", +] diff --git a/components/src/dynamo/aisimulate/output/dgdr_run/snapshot.py b/components/src/dynamo/aisimulate/output/dgdr_run/snapshot.py new file mode 100644 index 000000000000..f14ff16264ed --- /dev/null +++ b/components/src/dynamo/aisimulate/output/dgdr_run/snapshot.py @@ -0,0 +1,148 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Desired-state snapshot of one DGDRRun, written to the pod-local volume. + +The Sweeper container atomically replaces a single YAML file with the complete +*current* projection of one ``DynamoGraphDeploymentRun``; the publisher +sidecar in the same pod consumes it directly (shared ``emptyDir``). Nothing +here involves a ConfigMap, the Dynamo event plane, or any cross-pod RPC. + +The snapshot is desired state, not a command list and not history: + +* ``run`` carries progress and, once ``terminal`` is true, the final outcome; +* ``candidates`` is the bounded set of evaluated points that should exist as + ``DynamoGraphDeploymentCandidate`` objects, best-first for scalar searches. + +Each candidate has a stable ``id`` that depends only on the evaluated point +(its resolved parameters), never on its list position or rank. Reordering +the list must therefore never change an identity or a rendered manifest; it +only changes the ordered references the publisher writes to the run status. + +Each published snapshot is complete. The publisher only needs the latest one, +so intermediate states may be coalesced away, but a terminal snapshot must +always be written (see ``DGDRRunOutputAdapter.close``). +""" + +from __future__ import annotations + +import time +from dataclasses import dataclass, field +from enum import Enum +from pathlib import Path +from typing import Any + +import yaml + +from dynamo.aisimulate.output.dgd.writers.atomic import replace_text + +SNAPSHOT_FILE_NAME = "dgdr_run_snapshot.yaml" +SCHEMA_VERSION = 1 + + +class RunPhase(str, Enum): + """Run lifecycle as reported by the producer. Anything but RUNNING is terminal.""" + + RUNNING = "Running" + SUCCEEDED = "Succeeded" + FAILED = "Failed" + + @property + def terminal(self) -> bool: + return self is not RunPhase.RUNNING + + +class CandidateOutcome(str, Enum): + MATERIALIZED = "materialized" + MATERIALIZATION_FAILED = "materialization_failed" + + +@dataclass(frozen=True) +class SnapshotCandidate: + """One evaluated point in the bounded projection. + + ``parameters`` and ``metrics`` are the immutable evaluation facts the + publisher needs to build the candidate's resource: a DGD manifest alone + does not identify an evaluated point, because two evaluations can render + the same DGD under different parameters and still have different metrics. + """ + + id: str + outcome: CandidateOutcome + parameters: dict[str, Any] = field(default_factory=dict) + metrics: dict[str, Any] = field(default_factory=dict) + manifest: str | None = None + error: str | None = None + + def __post_init__(self) -> None: + if not self.id: + raise ValueError("candidate id must not be empty") + if self.outcome is CandidateOutcome.MATERIALIZED: + if not self.manifest: + raise ValueError("a materialized candidate must carry its manifest") + if self.error is not None: + raise ValueError("a materialized candidate must not carry an error") + elif not self.error: + raise ValueError("a failed candidate must carry its error") + elif self.manifest is not None: + raise ValueError("a failed candidate must not carry a manifest") + + def to_dict(self) -> dict[str, Any]: + payload: dict[str, Any] = { + "id": self.id, + "outcome": self.outcome.value, + "parameters": self.parameters, + "metrics": self.metrics, + } + if self.manifest is not None: + payload["manifest"] = self.manifest + if self.error is not None: + payload["error"] = self.error + return payload + + +@dataclass(frozen=True) +class DGDRRunSnapshot: + """The complete current projection of one run.""" + + phase: RunPhase + round_no: int + evaluated: int + candidates: tuple[SnapshotCandidate, ...] = () + message: str = "" + error: str = "" + + def to_dict(self) -> dict[str, Any]: + run: dict[str, Any] = { + "phase": self.phase.value, + "terminal": self.phase.terminal, + } + if self.message: + run["message"] = self.message + if self.error: + run["error"] = self.error + return { + "schemaVersion": SCHEMA_VERSION, + "timestamp": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + "run": run, + "progress": {"round": self.round_no, "evaluated": self.evaluated}, + "candidates": [candidate.to_dict() for candidate in self.candidates], + } + + +def write_snapshot(directory: Path, snapshot: DGDRRunSnapshot) -> Path: + """Atomically replace the snapshot file; readers never see a partial file.""" + path = directory / SNAPSHOT_FILE_NAME + replace_text(path, yaml.safe_dump(snapshot.to_dict(), sort_keys=False)) + return path + + +__all__ = [ + "SCHEMA_VERSION", + "SNAPSHOT_FILE_NAME", + "CandidateOutcome", + "DGDRRunSnapshot", + "RunPhase", + "SnapshotCandidate", + "write_snapshot", +] diff --git a/components/src/dynamo/aisimulate/tests/dgdr_run/test_adapter.py b/components/src/dynamo/aisimulate/tests/dgdr_run/test_adapter.py new file mode 100644 index 000000000000..bdb2df7dae3c --- /dev/null +++ b/components/src/dynamo/aisimulate/tests/dgdr_run/test_adapter.py @@ -0,0 +1,884 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import threading +import time +from dataclasses import dataclass +from pathlib import Path +from types import SimpleNamespace +from typing import Any + +import pytest +import yaml + +from dynamo.aisimulate.output.dgd.renderers import CandidateMaterializationError +from dynamo.aisimulate.output.dgdr_run import adapter as adapter_module +from dynamo.aisimulate.output.dgdr_run.adapter import ( + DGDRRunOutputAdapter, + DGDRRunOutputConfig, + candidate_id_for, + create_adapter, +) +from dynamo.aisimulate.output.dgdr_run.snapshot import SNAPSHOT_FILE_NAME, RunPhase + +pytestmark = [ + pytest.mark.unit, + pytest.mark.gpu_0, + pytest.mark.pre_merge, + pytest.mark.planner, + pytest.mark.parallel, +] + + +@dataclass +class Record: + candidate_id: str + score: float | None + used_gpus: int | None + config: dict[str, Any] + status: str = "feasible" + objectives: dict[str, float] | None = None + + +def record(tag: str, score: float, gpus: int = 8, **kwargs: Any) -> Record: + return Record( + candidate_id=f"sweeper-{tag}", + score=score, + used_gpus=gpus, + config={"backend": "vllm", "backend_version": "0.1", "tag": tag}, + **kwargs, + ) + + +def make_config(tmp_path: Path, **overrides: Any) -> DGDRRunOutputConfig: + values: dict[str, Any] = { + "name": "sweep", + "snapshot_dir": str(tmp_path), + "runtime_image": "nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.2.3", + "num_gpus_per_node": 8, + "max_candidates": 3, + "snapshot_interval_seconds": 0, + } + values.update(overrides) + return DGDRRunOutputConfig(**values) + + +def load(tmp_path: Path) -> dict[str, Any]: + return yaml.safe_load((tmp_path / SNAPSHOT_FILE_NAME).read_text()) + + +def manifests_by_id(snapshot: dict[str, Any]) -> dict[str, str]: + return {c["id"]: c["manifest"] for c in snapshot["candidates"]} + + +def tags(snapshot: dict[str, Any]) -> list[str]: + return [candidate["parameters"]["tag"] for candidate in snapshot["candidates"]] + + +def wait_for(predicate: Any, timeout: float = 5.0) -> None: + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if predicate(): + return + time.sleep(0.01) + raise AssertionError("condition not reached before the timeout") + + +@pytest.fixture +def renders(monkeypatch: pytest.MonkeyPatch) -> list[str]: + """Replace the shared materializer with a counting fake.""" + calls: list[str] = [] + + def fake_render(candidate: Any, workload: Any, options: Any, **kwargs: Any) -> str: + tag = candidate.config["tag"] + calls.append(tag) + if tag.startswith("bad"): + raise CandidateMaterializationError(f"cannot render {tag}") + return ( + "apiVersion: nvidia.com/v1beta1\n" + "kind: DynamoGraphDeployment\n" + f"metadata:\n name: {kwargs['dgd_name']}\n" + f"spec:\n marker: {tag}\n" + ) + + monkeypatch.setattr(adapter_module, "render_dgd", fake_render) + return calls + + +# -- projection --------------------------------------------------------------- + + +def test_start_writes_the_initial_snapshot_immediately( + tmp_path: Path, renders: list[str] +) -> None: + config = make_config(tmp_path, snapshot_interval_seconds=60) + with DGDRRunOutputAdapter(config, workload=None): + snapshot = load(tmp_path) + assert snapshot["run"] == {"phase": "Running", "terminal": False} + assert snapshot["candidates"] == [] + assert snapshot["progress"] == {"round": 0, "evaluated": 0} + + +def test_projection_is_bounded_and_best_first_whatever_is_evaluated( + tmp_path: Path, renders: list[str] +) -> None: + adapter = DGDRRunOutputAdapter(make_config(tmp_path), workload=None) + for index in range(500): + adapter.on_candidate(record(f"p{index:03d}", score=float(index))) + adapter.close() + + snapshot = load(tmp_path) + assert tags(snapshot) == ["p499", "p498", "p497"] + assert snapshot["progress"]["evaluated"] == 500 + # Only retained points are ever materialized, never all 500. + assert len(renders) == 3 + + +def test_ties_break_on_fewer_gpus_then_id_deterministically( + tmp_path: Path, renders: list[str] +) -> None: + adapter = DGDRRunOutputAdapter(make_config(tmp_path, max_candidates=4), None) + adapter.on_candidate(record("big", score=5.0, gpus=16)) + adapter.on_candidate(record("small", score=5.0, gpus=8)) + adapter.on_candidate(record("best", score=9.0, gpus=64)) + adapter.close() + + assert tags(load(tmp_path)) == ["best", "small", "big"] + + +def test_infeasible_candidates_are_counted_but_never_retained( + tmp_path: Path, renders: list[str] +) -> None: + adapter = DGDRRunOutputAdapter(make_config(tmp_path), workload=None) + adapter.on_candidate(record("ok", 1.0)) + adapter.on_candidate(record("nope", 99.0, status="infeasible")) + adapter.close() + + snapshot = load(tmp_path) + assert tags(snapshot) == ["ok"] + assert snapshot["progress"]["evaluated"] == 2 + + +def test_snapshot_preserves_the_evaluated_point_not_just_its_manifest( + tmp_path: Path, renders: list[str] +) -> None: + adapter = DGDRRunOutputAdapter(make_config(tmp_path), workload=None) + adapter.on_candidate(record("a", 87.7, gpus=4, objectives={"goodput": 12.5})) + adapter.close() + + (candidate,) = load(tmp_path)["candidates"] + assert candidate["outcome"] == "materialized" + assert candidate["parameters"] == { + "backend": "vllm", + "backend_version": "0.1", + "tag": "a", + } + assert candidate["metrics"] == { + "score": 87.7, + "usedGpus": 4, + "objectives": {"goodput": 12.5}, + } + assert "marker: a" in candidate["manifest"] + + +def test_round_progress_is_monotonic(tmp_path: Path, renders: list[str]) -> None: + adapter = DGDRRunOutputAdapter(make_config(tmp_path), workload=None) + adapter.on_round(3, []) + adapter.on_round(2, []) + adapter.close() + assert load(tmp_path)["progress"]["round"] == 3 + + +# -- identity, immutability, materialization ---------------------------------- + + +def test_identity_and_manifests_do_not_depend_on_rank_or_observation_order( + tmp_path: Path, renders: list[str] +) -> None: + first = tmp_path / "first" + second = tmp_path / "second" + one = DGDRRunOutputAdapter(make_config(first), workload=None) + one.on_candidate(record("a", 1.0)) + one.on_candidate(record("b", 2.0)) + one.close() + # Same two points, opposite scores and opposite observation order. + two = DGDRRunOutputAdapter(make_config(second), workload=None) + two.on_candidate(record("b", 1.0)) + two.on_candidate(record("a", 2.0)) + two.close() + + one_snapshot, two_snapshot = load(first), load(second) + assert tags(one_snapshot) == ["b", "a"] + assert tags(two_snapshot) == ["a", "b"] + assert manifests_by_id(one_snapshot) == manifests_by_id(two_snapshot) + + +def test_identity_depends_only_on_the_resolved_parameters() -> None: + assert candidate_id_for({"a": 1, "b": 2}) == candidate_id_for({"b": 2, "a": 1}) + assert candidate_id_for({"a": 1}) != candidate_id_for({"a": 2}) + assert candidate_id_for({"a": 1}).startswith("evaluated-point-") + + +def test_a_new_better_point_reorders_but_does_not_rematerialize_retained_ones( + tmp_path: Path, renders: list[str] +) -> None: + adapter = DGDRRunOutputAdapter(make_config(tmp_path), workload=None) + adapter.start() + adapter.on_candidate(record("a", 1.0)) + adapter.on_candidate(record("b", 2.0)) + wait_for(lambda: tags(load(tmp_path)) == ["b", "a"]) + before = {c["id"]: c["manifest"] for c in load(tmp_path)["candidates"]} + assert sorted(renders) == ["a", "b"] + + adapter.on_candidate(record("c", 3.0)) + wait_for(lambda: tags(load(tmp_path)) == ["c", "b", "a"]) + adapter.close() + + after = {c["id"]: c["manifest"] for c in load(tmp_path)["candidates"]} + assert {key: after[key] for key in before} == before + assert sorted(renders) == ["a", "b", "c"] # only the new point was rendered + + +def test_progress_updates_do_not_rematerialize( + tmp_path: Path, renders: list[str] +) -> None: + adapter = DGDRRunOutputAdapter(make_config(tmp_path), workload=None) + adapter.start() + adapter.on_candidate(record("a", 1.0)) + wait_for(lambda: tags(load(tmp_path)) == ["a"]) + for round_no in range(1, 20): + adapter.on_round(round_no, []) + wait_for(lambda: load(tmp_path)["progress"]["round"] == 19) + adapter.close() + assert renders == ["a"] + + +def test_a_point_is_immutable_once_observed(tmp_path: Path, renders: list[str]) -> None: + adapter = DGDRRunOutputAdapter(make_config(tmp_path), workload=None) + adapter.on_candidate(record("a", 1.0)) + adapter.on_candidate(record("a", 50.0)) # same parameters, noisy re-evaluation + adapter.close() + + (candidate,) = load(tmp_path)["candidates"] + assert candidate["metrics"]["score"] == 1.0 + assert load(tmp_path)["progress"]["evaluated"] == 2 + + +def test_a_persistent_materialization_failure_is_reported_and_retried_a_bounded_number_of_times( + tmp_path: Path, renders: list[str] +) -> None: + adapter = DGDRRunOutputAdapter(make_config(tmp_path), workload=None) + adapter.start() + adapter.on_candidate(record("bad1", 5.0)) + wait_for(lambda: len(load(tmp_path)["candidates"]) == 1) + for round_no in range(1, 10): + adapter.on_round(round_no, []) + wait_for(lambda: load(tmp_path)["progress"]["round"] == 9) + adapter.close() + + (candidate,) = load(tmp_path)["candidates"] + assert candidate["outcome"] == "materialization_failed" + assert "cannot render bad1" in candidate["error"] + assert "manifest" not in candidate + assert renders == ["bad1"] * adapter_module._MAX_RENDER_ATTEMPTS + + +# -- rate limiting and coalescing --------------------------------------------- + + +def test_many_callbacks_in_one_interval_produce_one_periodic_write( + tmp_path: Path, renders: list[str] +) -> None: + config = make_config(tmp_path, snapshot_interval_seconds=0.5) + adapter = DGDRRunOutputAdapter(config, workload=None) + adapter.start() + assert adapter.snapshot_writes == 1 # initial + + for index in range(300): + adapter.on_candidate(record(f"p{index:03d}", score=float(index))) + time.sleep(0.15) + assert adapter.snapshot_writes == 1 # still inside the interval + + wait_for(lambda: adapter.snapshot_writes == 2, timeout=5) + snapshot = load(tmp_path) + # The one periodic write carries the latest state, not an arbitrary + # intermediate one. + assert snapshot["progress"]["evaluated"] == 300 + assert tags(snapshot) == ["p299", "p298", "p297"] + + time.sleep(0.2) + assert adapter.snapshot_writes == 2 + adapter.close() + + +def test_terminal_state_bypasses_the_rate_limit_and_is_always_flushed( + tmp_path: Path, renders: list[str] +) -> None: + config = make_config(tmp_path, snapshot_interval_seconds=3600) + adapter = DGDRRunOutputAdapter(config, workload=None) + adapter.start() + for index in range(100): + adapter.on_candidate(record(f"p{index:03d}", score=float(index))) + + started = time.monotonic() + adapter.close(phase=RunPhase.FAILED, message="boom", error="OOM") + assert time.monotonic() - started < 5 + + snapshot = load(tmp_path) + assert snapshot["run"] == { + "phase": "Failed", + "terminal": True, + "message": "boom", + "error": "OOM", + } + assert tags(snapshot) == ["p099", "p098", "p097"] + assert snapshot["progress"]["evaluated"] == 100 + assert adapter.snapshot_writes == 2 # initial + terminal, nothing in between + + +def test_the_terminal_snapshot_contains_every_update_accepted_before_close( + tmp_path: Path, renders: list[str] +) -> None: + config = make_config(tmp_path, snapshot_interval_seconds=3600) + with DGDRRunOutputAdapter(config, workload=None) as adapter: + for index in range(50): + adapter.on_candidate(record(f"w{index}", score=1.0 + index)) + adapter.on_candidate(record("winner", score=1000.0)) # right before close + + snapshot = load(tmp_path) + assert snapshot["run"]["terminal"] is True + assert tags(snapshot)[0] == "winner" + + +def test_callbacks_never_wait_for_a_slow_writer( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + release = threading.Event() + + def slow_render(candidate: Any, *args: Any, **kwargs: Any) -> str: + release.wait(5) + return "kind: DynamoGraphDeployment\n" + + monkeypatch.setattr(adapter_module, "render_dgd", slow_render) + adapter = DGDRRunOutputAdapter(make_config(tmp_path), workload=None) + adapter.start() + adapter.on_candidate(record("a", 1.0)) + time.sleep(0.1) # the writer is now stuck rendering + + started = time.monotonic() + for index in range(1000): + adapter.on_candidate(record(f"q{index}", score=float(index))) + assert time.monotonic() - started < 2 + release.set() + adapter.close() + # Nothing was dropped while the writer was stuck. + assert load(tmp_path)["progress"]["evaluated"] == 1001 + + +def test_updates_after_close_are_ignored_and_close_is_idempotent( + tmp_path: Path, renders: list[str] +) -> None: + adapter = DGDRRunOutputAdapter(make_config(tmp_path), workload=None) + adapter.on_candidate(record("a", 1.0)) + adapter.close(phase=RunPhase.SUCCEEDED) + written = (tmp_path / SNAPSHOT_FILE_NAME).read_text() + writes = adapter.snapshot_writes + + adapter.on_candidate(record("late", 99.0)) + adapter.on_round(9, []) + adapter.close(phase=RunPhase.FAILED, error="ignored") + + assert (tmp_path / SNAPSHOT_FILE_NAME).read_text() == written + assert adapter.snapshot_writes == writes + + +def test_close_works_when_start_was_never_called( + tmp_path: Path, renders: list[str] +) -> None: + adapter = DGDRRunOutputAdapter(make_config(tmp_path), workload=None) + adapter.close(phase=RunPhase.FAILED, error="setup failed") + assert load(tmp_path)["run"]["phase"] == "Failed" + + +def test_close_rejects_a_non_terminal_phase(tmp_path: Path) -> None: + adapter = DGDRRunOutputAdapter(make_config(tmp_path), workload=None) + with pytest.raises(ValueError, match="terminal"): + adapter.close(phase=RunPhase.RUNNING) + + +# -- failure handling --------------------------------------------------------- + + +def test_context_manager_records_success_and_failure( + tmp_path: Path, renders: list[str] +) -> None: + ok = tmp_path / "ok" + with DGDRRunOutputAdapter(make_config(ok), workload=None): + pass + assert load(ok)["run"]["phase"] == "Succeeded" + + failed = tmp_path / "failed" + with pytest.raises(RuntimeError, match="search crashed"): + with DGDRRunOutputAdapter(make_config(failed), workload=None): + raise RuntimeError("search crashed") + snapshot = load(failed) + assert snapshot["run"]["phase"] == "Failed" + assert snapshot["run"]["error"] == "RuntimeError" + + +def test_a_terminal_write_failure_is_raised_not_swallowed( + tmp_path: Path, renders: list[str], monkeypatch: pytest.MonkeyPatch +) -> None: + adapter = DGDRRunOutputAdapter(make_config(tmp_path), workload=None) + adapter.start() + + def broken(*_: Any, **__: Any) -> None: + raise OSError("disk full") + + monkeypatch.setattr(adapter_module, "write_snapshot", broken) + with pytest.raises(OSError, match="disk full"): + adapter.close() + + +def test_a_failed_periodic_write_is_retried_with_the_latest_state( + tmp_path: Path, renders: list[str], monkeypatch: pytest.MonkeyPatch +) -> None: + adapter = DGDRRunOutputAdapter(make_config(tmp_path), workload=None) + adapter.start() + real = adapter_module.write_snapshot + attempts = {"count": 0} + + def flaky(directory: Path, snapshot: Any) -> Path: + attempts["count"] += 1 + if attempts["count"] == 1: + raise OSError("transient") + return real(directory, snapshot) + + monkeypatch.setattr(adapter_module, "write_snapshot", flaky) + adapter.on_candidate(record("a", 1.0)) + wait_for(lambda: tags(load(tmp_path)) == ["a"]) + adapter.close() + assert attempts["count"] >= 2 + + +def test_a_candidate_that_cannot_be_read_does_not_break_the_search( + tmp_path: Path, renders: list[str] +) -> None: + adapter = DGDRRunOutputAdapter(make_config(tmp_path), workload=None) + adapter.on_candidate(SimpleNamespace(status="feasible")) # type: ignore[arg-type] + adapter.on_candidate(record("ok", 1.0)) + adapter.close() + assert tags(load(tmp_path)) == ["ok"] + + +# -- configuration and plugin ------------------------------------------------- + + +def test_pareto_objectives_are_rejected_explicitly(tmp_path: Path) -> None: + with pytest.raises(NotImplementedError, match="scalar"): + DGDRRunOutputAdapter(make_config(tmp_path), workload=None, is_pareto=True) + + +@pytest.mark.parametrize( + "overrides", + [ + {"max_candidates": 0}, + {"snapshot_interval_seconds": -1}, + {"name": "Not_DNS"}, + {"num_gpus_per_node": 0}, + {"unknown_field": 1}, + ], +) +def test_invalid_configuration_is_rejected( + tmp_path: Path, overrides: dict[str, Any] +) -> None: + with pytest.raises(ValueError): + make_config(tmp_path, **overrides) + + +def test_there_is_no_hardcoded_candidate_ceiling( + tmp_path: Path, renders: list[str] +) -> None: + adapter = DGDRRunOutputAdapter(make_config(tmp_path, max_candidates=200), None) + for index in range(150): + adapter.on_candidate(record(f"p{index:03d}", score=float(index))) + adapter.close() + assert len(load(tmp_path)["candidates"]) == 150 + + +def test_a_snapshot_directory_is_required() -> None: + config = DGDRRunOutputConfig( + name="sweep", + runtime_image="nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.2.3", + num_gpus_per_node=8, + ) + with pytest.raises(ValueError, match="snapshot directory"): + DGDRRunOutputAdapter(config, workload=None) + + +def test_the_plugin_is_discoverable_and_builds_a_live_adapter(tmp_path: Path) -> None: + plugin = create_adapter() + assert plugin.name == "dgdr_run" + live = plugin.live( + { + "name": "sweep", + "snapshot_dir": str(tmp_path), + "runtime_image": "nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.2.3", + "num_gpus_per_node": 8, + }, + workload=None, + ) + assert isinstance(live, DGDRRunOutputAdapter) + + +def test_plugin_subscribe_validates_config_before_search() -> None: + plugin = adapter_module.DGDRRunOutputPlugin() + good = { + "name": "run", + "runtime_image": "nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.2.3", + "num_gpus_per_node": 8, + } + + assert plugin.subscribe(good) is None + with pytest.raises(ValueError, match="num_gpus_per_node"): + plugin.subscribe({**good, "num_gpus_per_node": 0}) + with pytest.raises(ValueError, match="canonical"): + plugin.subscribe( + {**good, "runtime_image": "nvcr.io/nvidia/ai-dynamo/vllm-runtime:latest"} + ) + + +def test_plugin_write_publishes_the_final_selection_as_a_terminal_snapshot( + tmp_path: Path, renders: list[str], monkeypatch: pytest.MonkeyPatch +) -> None: + class FakeSearchConfig: + @staticmethod + def model_validate(_: Any) -> Any: + return SimpleNamespace(workload="workload") + + monkeypatch.setattr(adapter_module, "SmartSearchConfig", FakeSearchConfig) + result = SimpleNamespace( + views=SimpleNamespace(pareto_front=[]), + provenance=SimpleNamespace(config={}), + selected_candidates=[ + record("a", 9.0), + record("b", 5.0), + ], + ) + written = create_adapter().write( + { + "name": "sweep", + "runtime_image": "nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.2.3", + "num_gpus_per_node": 8, + }, + result=result, # type: ignore[arg-type] + output_dir=tmp_path, + ) + + assert written == [Path(SNAPSHOT_FILE_NAME)] + snapshot = load(tmp_path) + assert snapshot["run"]["terminal"] is True + assert tags(snapshot) == ["a", "b"] + + +def test_plugin_write_refuses_pareto_and_empty_results(tmp_path: Path) -> None: + config = { + "name": "sweep", + "runtime_image": "nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.2.3", + "num_gpus_per_node": 8, + } + pareto = SimpleNamespace(views=SimpleNamespace(pareto_front=[object()])) + with pytest.raises(NotImplementedError): + create_adapter().write(config, result=pareto, output_dir=tmp_path) # type: ignore[arg-type] + empty = SimpleNamespace( + views=SimpleNamespace(pareto_front=[]), selected_candidates=[] + ) + with pytest.raises(CandidateMaterializationError): + create_adapter().write(config, result=empty, output_dir=tmp_path) # type: ignore[arg-type] + + +PLUGIN_CONFIG = { + "name": "sweep", + "runtime_image": "nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.2.3", + "num_gpus_per_node": 8, + "snapshot_interval_seconds": 0, +} + + +def test_plugin_subscribe_with_a_context_publishes_live( + tmp_path: Path, renders: list[str] +) -> None: + config = {**PLUGIN_CONFIG, "snapshot_dir": str(tmp_path)} + callbacks = create_adapter().subscribe( + config, context=SimpleNamespace(workload="workload") + ) + + assert callbacks is not None + assert callbacks.on_candidate is not None + assert callbacks.on_round is not None + # The initial snapshot exists before the first candidate is evaluated. + assert load(tmp_path)["run"] == {"phase": "Running", "terminal": False} + + callbacks.on_candidate(record("a", 9.0)) + wait_for(lambda: tags(load(tmp_path)) == ["a"]) + + +def test_plugin_subscribe_without_context_or_snapshot_dir_only_validates( + tmp_path: Path, +) -> None: + plugin = create_adapter() + with_dir = {**PLUGIN_CONFIG, "snapshot_dir": str(tmp_path)} + assert plugin.subscribe(with_dir) is None + assert plugin.subscribe(with_dir, context=SimpleNamespace(workload=None)) is None + assert ( + plugin.subscribe(PLUGIN_CONFIG, context=SimpleNamespace(workload="workload")) + is None + ) + assert not (tmp_path / SNAPSHOT_FILE_NAME).exists() + + +def _final_result(evaluated: int = 42) -> Any: + return SimpleNamespace( + views=SimpleNamespace(pareto_front=[]), + provenance=SimpleNamespace(config={}), + selected_candidates=[record("a", 9.0)], + counts=SimpleNamespace(evaluated=evaluated), + ) + + +@pytest.fixture +def fake_search_config(monkeypatch: pytest.MonkeyPatch) -> None: + class FakeSearchConfig: + @staticmethod + def model_validate(_: Any) -> Any: + return SimpleNamespace(workload="workload") + + monkeypatch.setattr(adapter_module, "SmartSearchConfig", FakeSearchConfig) + + +def test_plugin_write_records_progress_from_the_result_counts( + tmp_path: Path, renders: list[str], fake_search_config: None +) -> None: + create_adapter().write(PLUGIN_CONFIG, result=_final_result(42), output_dir=tmp_path) + assert load(tmp_path)["progress"]["evaluated"] == 42 + + +def test_plugin_write_reports_no_artifact_for_a_snapshot_outside_output_dir( + tmp_path: Path, renders: list[str], fake_search_config: None +) -> None: + shared = tmp_path / "shared" + shared.mkdir() + out = tmp_path / "out" + out.mkdir() + + written = create_adapter().write( + {**PLUGIN_CONFIG, "snapshot_dir": str(shared)}, + result=_final_result(), + output_dir=out, + ) + + assert written == [] + assert load(shared)["run"]["terminal"] is True + assert not (out / SNAPSHOT_FILE_NAME).exists() + + +def test_a_transient_materialization_failure_is_retried_and_recovers( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + attempts: list[int] = [] + + def flaky_render(candidate: Any, workload: Any, options: Any, **kwargs: Any) -> str: + attempts.append(1) + if len(attempts) == 1: + raise CandidateMaterializationError("cold start") + return "apiVersion: nvidia.com/v1beta1\nkind: DynamoGraphDeployment\n" + + monkeypatch.setattr(adapter_module, "render_dgd", flaky_render) + adapter = DGDRRunOutputAdapter(make_config(tmp_path), workload=None) + adapter.start() + adapter.on_candidate(record("a", 5.0)) + wait_for(lambda: len(load(tmp_path)["candidates"]) == 1) + assert load(tmp_path)["candidates"][0]["outcome"] == "materialization_failed" + + adapter.on_round(1, []) # any later write retries the failed render + wait_for(lambda: load(tmp_path)["candidates"][0]["outcome"] == "materialized") + adapter.close() + + (candidate,) = load(tmp_path)["candidates"] + assert candidate["outcome"] == "materialized" + assert "error" not in candidate + assert len(attempts) == 2 + + +def test_a_materialized_candidate_is_never_rendered_again( + tmp_path: Path, renders: list[str] +) -> None: + adapter = DGDRRunOutputAdapter(make_config(tmp_path), workload=None) + adapter.start() + adapter.on_candidate(record("a", 5.0)) + wait_for(lambda: len(load(tmp_path)["candidates"]) == 1) + for round_no in range(1, 6): + adapter.on_round(round_no, []) + wait_for(lambda: load(tmp_path)["progress"]["round"] == 5) + adapter.close() + assert renders == ["a"] + + +def test_plugin_write_keeps_the_round_recorded_by_the_live_snapshot( + tmp_path: Path, renders: list[str], fake_search_config: None +) -> None: + live = create_adapter().subscribe( + {**PLUGIN_CONFIG, "snapshot_dir": str(tmp_path)}, + context=SimpleNamespace(workload="workload"), + ) + assert live is not None and live.on_round is not None + live.on_round(3, []) + wait_for(lambda: load(tmp_path)["progress"]["round"] == 3) + + create_adapter().write( + {**PLUGIN_CONFIG, "snapshot_dir": str(tmp_path)}, + result=_final_result(4), + output_dir=tmp_path, + ) + + snapshot = load(tmp_path) + assert snapshot["run"]["terminal"] is True + assert snapshot["progress"] == {"round": 3, "evaluated": 4} + + +def test_plugin_write_without_a_live_snapshot_starts_at_round_zero( + tmp_path: Path, renders: list[str], fake_search_config: None +) -> None: + create_adapter().write(PLUGIN_CONFIG, result=_final_result(2), output_dir=tmp_path) + assert load(tmp_path)["progress"] == {"round": 0, "evaluated": 2} + + +# -- lifecycle: how the search ended -------------------------------------------- + + +def _subscribed(tmp_path: Path) -> Any: + callbacks = create_adapter().subscribe( + {**PLUGIN_CONFIG, "snapshot_dir": str(tmp_path)}, + context=SimpleNamespace(workload="workload"), + ) + assert callbacks is not None + return callbacks + + +def test_subscribe_registers_the_lifecycle_callbacks( + tmp_path: Path, renders: list[str] +) -> None: + callbacks = _subscribed(tmp_path) + assert callbacks.on_complete is not None + assert callbacks.on_failure is not None + + +def test_a_failed_search_leaves_a_terminal_failed_snapshot( + tmp_path: Path, renders: list[str] +) -> None: + callbacks = _subscribed(tmp_path) + callbacks.on_candidate(record("a", 9.0)) + callbacks.on_round(2, []) + + callbacks.on_failure(RuntimeError("replay crashed")) + + snapshot = load(tmp_path) + assert snapshot["run"] == { + "phase": "Failed", + "terminal": True, + "message": "replay crashed", + "error": "RuntimeError", + } + assert tags(snapshot) == ["a"] + assert snapshot["progress"]["round"] == 2 + + +def test_a_long_failure_message_is_truncated( + tmp_path: Path, renders: list[str] +) -> None: + callbacks = _subscribed(tmp_path) + callbacks.on_failure(RuntimeError("x" * 5000)) + assert len(load(tmp_path)["run"]["message"]) == adapter_module._MAX_ERROR_MESSAGE + + +def test_a_search_that_finds_nothing_is_recorded_as_failed( + tmp_path: Path, renders: list[str] +) -> None: + callbacks = _subscribed(tmp_path) + callbacks.on_candidate(record("infeasible", 1.0, status="infeasible")) + + callbacks.on_complete() + + run = load(tmp_path)["run"] + assert run["phase"] == "Failed" + assert run["terminal"] is True + assert run["message"] == "no feasible candidate found" + + +def test_a_completed_search_flushes_stops_the_writer_and_stays_non_terminal( + tmp_path: Path, renders: list[str] +) -> None: + config = make_config(tmp_path, snapshot_interval_seconds=60) + adapter = DGDRRunOutputAdapter(config, workload=None) + adapter.start() + adapter.on_candidate(record("a", 9.0)) + adapter.on_round(3, []) + + adapter.finish_search() + + snapshot = load(tmp_path) + assert snapshot["run"] == {"phase": "Running", "terminal": False} + assert tags(snapshot) == ["a"] + assert snapshot["progress"] == {"round": 3, "evaluated": 1} + # The writer is gone: nothing can overwrite the terminal snapshot written next. + assert adapter._thread is not None and not adapter._thread.is_alive() + writes = adapter.snapshot_writes + adapter.on_round(4, []) + time.sleep(0.1) + assert adapter.snapshot_writes == writes + + +def test_the_terminal_snapshot_after_a_completed_search_keeps_the_live_progress( + tmp_path: Path, renders: list[str], fake_search_config: None +) -> None: + callbacks = _subscribed(tmp_path) + callbacks.on_candidate(record("a", 9.0)) + callbacks.on_round(3, []) + callbacks.on_complete() + + create_adapter().write( + {**PLUGIN_CONFIG, "snapshot_dir": str(tmp_path)}, + result=_final_result(1), + output_dir=tmp_path, + ) + + snapshot = load(tmp_path) + assert snapshot["run"]["phase"] == "Succeeded" + assert snapshot["progress"] == {"round": 3, "evaluated": 1} + + +def test_failure_after_completion_is_a_noop_and_completion_after_failure_too( + tmp_path: Path, renders: list[str] +) -> None: + callbacks = _subscribed(tmp_path) + callbacks.on_candidate(record("a", 9.0)) + callbacks.on_failure(RuntimeError("first")) + callbacks.on_complete() + callbacks.on_failure(RuntimeError("second")) + assert load(tmp_path)["run"]["message"] == "first" + + +def test_subscribe_still_works_with_an_aisimulate_without_lifecycle_callbacks( + tmp_path: Path, renders: list[str], monkeypatch: pytest.MonkeyPatch +) -> None: + @dataclass(frozen=True) + class LegacyCallbacks: + on_candidate: Any = None + on_round: Any = None + + monkeypatch.setattr( + adapter_module, "RecommendationOutputCallbacks", LegacyCallbacks + ) + callbacks = _subscribed(tmp_path) + assert isinstance(callbacks, LegacyCallbacks) + assert callbacks.on_candidate is not None diff --git a/components/src/dynamo/aisimulate/tests/dgdr_run/test_snapshot.py b/components/src/dynamo/aisimulate/tests/dgdr_run/test_snapshot.py new file mode 100644 index 000000000000..107f79a8e0a6 --- /dev/null +++ b/components/src/dynamo/aisimulate/tests/dgdr_run/test_snapshot.py @@ -0,0 +1,154 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from pathlib import Path + +import pytest +import yaml + +from dynamo.aisimulate.output.dgdr_run.snapshot import ( + SCHEMA_VERSION, + SNAPSHOT_FILE_NAME, + CandidateOutcome, + DGDRRunSnapshot, + RunPhase, + SnapshotCandidate, + write_snapshot, +) + +pytestmark = [ + pytest.mark.unit, + pytest.mark.gpu_0, + pytest.mark.pre_merge, + pytest.mark.planner, + pytest.mark.parallel, +] + + +def materialized(candidate_id: str = "evaluated-point-a") -> SnapshotCandidate: + return SnapshotCandidate( + id=candidate_id, + outcome=CandidateOutcome.MATERIALIZED, + parameters={"tp": 4}, + metrics={"score": 1.5, "usedGpus": 4}, + manifest="kind: DynamoGraphDeployment\n", + ) + + +def test_only_running_is_a_non_terminal_phase() -> None: + assert not RunPhase.RUNNING.terminal + assert RunPhase.SUCCEEDED.terminal + assert RunPhase.FAILED.terminal + + +def test_wire_format_of_a_running_snapshot() -> None: + payload = DGDRRunSnapshot( + phase=RunPhase.RUNNING, + round_no=7, + evaluated=48, + candidates=(materialized(),), + ).to_dict() + + assert payload["schemaVersion"] == SCHEMA_VERSION + assert payload["run"] == {"phase": "Running", "terminal": False} + assert payload["progress"] == {"round": 7, "evaluated": 48} + assert payload["candidates"] == [ + { + "id": "evaluated-point-a", + "outcome": "materialized", + "parameters": {"tp": 4}, + "metrics": {"score": 1.5, "usedGpus": 4}, + "manifest": "kind: DynamoGraphDeployment\n", + } + ] + assert payload["timestamp"].endswith("Z") + + +def test_terminal_snapshot_carries_the_outcome() -> None: + payload = DGDRRunSnapshot( + phase=RunPhase.FAILED, + round_no=2, + evaluated=9, + message="stopped", + error="OOM", + ).to_dict() + assert payload["run"] == { + "phase": "Failed", + "terminal": True, + "message": "stopped", + "error": "OOM", + } + assert payload["candidates"] == [] + + +def test_a_failed_candidate_carries_an_error_and_no_manifest() -> None: + failed = SnapshotCandidate( + id="evaluated-point-b", + outcome=CandidateOutcome.MATERIALIZATION_FAILED, + error="cannot render", + ) + assert failed.to_dict() == { + "id": "evaluated-point-b", + "outcome": "materialization_failed", + "parameters": {}, + "metrics": {}, + "error": "cannot render", + } + + +@pytest.mark.parametrize( + "kwargs", + [ + {"id": "", "outcome": CandidateOutcome.MATERIALIZED, "manifest": "x"}, + {"id": "a", "outcome": CandidateOutcome.MATERIALIZED}, + { + "id": "a", + "outcome": CandidateOutcome.MATERIALIZED, + "manifest": "x", + "error": "y", + }, + {"id": "a", "outcome": CandidateOutcome.MATERIALIZATION_FAILED}, + { + "id": "a", + "outcome": CandidateOutcome.MATERIALIZATION_FAILED, + "error": "y", + "manifest": "x", + }, + ], +) +def test_inconsistent_candidates_are_rejected(kwargs: dict) -> None: + with pytest.raises(ValueError): + SnapshotCandidate(**kwargs) + + +def test_write_replaces_the_file_atomically_and_leaves_no_temporary_files( + tmp_path: Path, +) -> None: + for round_no in range(5): + path = write_snapshot( + tmp_path, + DGDRRunSnapshot(phase=RunPhase.RUNNING, round_no=round_no, evaluated=0), + ) + assert path == tmp_path / SNAPSHOT_FILE_NAME + assert yaml.safe_load(path.read_text())["progress"]["round"] == round_no + + assert [p.name for p in tmp_path.iterdir()] == [SNAPSHOT_FILE_NAME] + + +def test_the_manifest_survives_a_yaml_round_trip_byte_for_byte( + tmp_path: Path, +) -> None: + manifest = "apiVersion: nvidia.com/v1beta1\nkind: DynamoGraphDeployment\nspec:\n a: 'x: y'\n" + candidate = SnapshotCandidate( + id="evaluated-point-c", + outcome=CandidateOutcome.MATERIALIZED, + manifest=manifest, + ) + path = write_snapshot( + tmp_path, + DGDRRunSnapshot( + phase=RunPhase.RUNNING, round_no=0, evaluated=1, candidates=(candidate,) + ), + ) + (loaded,) = yaml.safe_load(path.read_text())["candidates"] + assert loaded["manifest"] == manifest diff --git a/pyproject.toml b/pyproject.toml index ce60528c3d30..993eb9ceeb6b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -127,6 +127,7 @@ dynamo = "dynamo.replay.simulation:DynamoReplayRunnerFactory" [project.entry-points."aisimulate.output_adapters"] dgd = "dynamo.aisimulate.output.dgd:create_adapter" +dgdr_run = "dynamo.aisimulate.output.dgdr_run:create_adapter" [dependency-groups] docs = [ From 37ed714935a1c24e4622ad6f094cc76797804ebf Mon Sep 17 00:00:00 2001 From: devivasudevan <49675305+devivasudevan@users.noreply.github.com> Date: Wed, 7 Oct 2026 15:34:47 +0200 Subject: [PATCH 05/40] fix(aisimulate): keep Sweeper's selection order in dgdr_run; trim redundant tests --- .../aisimulate/output/dgdr_run/adapter.py | 26 ++++++++++--- .../aisimulate/tests/dgdr_run/test_adapter.py | 37 ++++++++----------- 2 files changed, 35 insertions(+), 28 deletions(-) diff --git a/components/src/dynamo/aisimulate/output/dgdr_run/adapter.py b/components/src/dynamo/aisimulate/output/dgdr_run/adapter.py index e8739203fd53..a3cf32f8b283 100644 --- a/components/src/dynamo/aisimulate/output/dgdr_run/adapter.py +++ b/components/src/dynamo/aisimulate/output/dgdr_run/adapter.py @@ -128,10 +128,15 @@ class _Retained: def rank_key(self) -> tuple[float, int, str]: """Ascending sort key: best first. - Highest score, then fewer GPUs, then id so that equal points order - deterministically. Matches Sweeper's own scalar ranking. + Mirrors Sweeper's scalar ``rank()``: highest score, then fewer GPUs, + then the canonical config JSON. It is only used to bound the live + retained set; the final snapshot keeps Sweeper's own selection order. """ - return (-self.score, self.used_gpus, self.id) + return ( + -self.score, + self.used_gpus, + json.dumps(self.parameters, sort_keys=True, separators=(",", ":")), + ) @dataclass(frozen=True) @@ -183,6 +188,7 @@ def __init__( *, snapshot_dir: Path | None = None, is_pareto: bool = False, + keep_arrival_order: bool = False, ) -> None: if is_pareto: raise NotImplementedError( @@ -218,6 +224,9 @@ def __init__( # holding _publish_lock. A failure is retried on later writes up to # _MAX_RENDER_ATTEMPTS in total, because a first render can fail # transiently (cold start); after that it is final. + # The final result is already ranked by Sweeper (goal and SLA aware); + # publishing it must not re-rank it. + self._keep_arrival_order = keep_arrival_order self._publish_lock = threading.Lock() self._materialized: dict[str, SnapshotCandidate] = {} self._render_attempts: dict[str, int] = {} @@ -401,7 +410,11 @@ def _publish(self) -> None: # Cleared together with the copy so no update is ever lost: an # update after this point sets the event again. self._wake.clear() - ordered = sorted(self._retained.values(), key=lambda r: r.rank_key) + ordered = ( + list(self._retained.values()) + if self._keep_arrival_order + else sorted(self._retained.values(), key=lambda r: r.rank_key) + ) round_no = self._round_no evaluated = self._evaluated terminal = self._terminal @@ -447,7 +460,6 @@ def _materialize(self, item: _Retained) -> SnapshotCandidate: manifest=manifest, ) except CandidateMaterializationError as exc: - # Retried on later writes until _MAX_RENDER_ATTEMPTS, then final. entry = SnapshotCandidate( id=item.id, outcome=CandidateOutcome.MATERIALIZATION_FAILED, @@ -534,7 +546,9 @@ def write( raise CandidateMaterializationError("no feasible candidate found") workload = SmartSearchConfig.model_validate(result.provenance.config).workload directory = Path(resolved.snapshot_dir or output_dir) - adapter = DGDRRunOutputAdapter(resolved, workload, snapshot_dir=directory) + adapter = DGDRRunOutputAdapter( + resolved, workload, snapshot_dir=directory, keep_arrival_order=True + ) for candidate in selected: adapter.on_candidate(_as_feasible(candidate)) counts = getattr(result, "counts", None) diff --git a/components/src/dynamo/aisimulate/tests/dgdr_run/test_adapter.py b/components/src/dynamo/aisimulate/tests/dgdr_run/test_adapter.py index bdb2df7dae3c..f076253a24fc 100644 --- a/components/src/dynamo/aisimulate/tests/dgdr_run/test_adapter.py +++ b/components/src/dynamo/aisimulate/tests/dgdr_run/test_adapter.py @@ -581,6 +581,21 @@ def model_validate(_: Any) -> Any: assert tags(snapshot) == ["a", "b"] +def test_plugin_write_keeps_sweepers_selection_order( + tmp_path: Path, renders: list[str], fake_search_config: None +) -> None: + # Sweeper's selection can be goal-dependent (e.g. fewest GPUs first), so the + # adapter must not re-rank it by score. + result = SimpleNamespace( + views=SimpleNamespace(pareto_front=[]), + provenance=SimpleNamespace(config={}), + selected_candidates=[record("low", 1.0), record("high", 9.0)], + counts=SimpleNamespace(evaluated=2), + ) + create_adapter().write(PLUGIN_CONFIG, result=result, output_dir=tmp_path) # type: ignore[arg-type] + assert tags(load(tmp_path)) == ["low", "high"] + + def test_plugin_write_refuses_pareto_and_empty_results(tmp_path: Path) -> None: config = { "name": "sweep", @@ -710,20 +725,6 @@ def flaky_render(candidate: Any, workload: Any, options: Any, **kwargs: Any) -> assert len(attempts) == 2 -def test_a_materialized_candidate_is_never_rendered_again( - tmp_path: Path, renders: list[str] -) -> None: - adapter = DGDRRunOutputAdapter(make_config(tmp_path), workload=None) - adapter.start() - adapter.on_candidate(record("a", 5.0)) - wait_for(lambda: len(load(tmp_path)["candidates"]) == 1) - for round_no in range(1, 6): - adapter.on_round(round_no, []) - wait_for(lambda: load(tmp_path)["progress"]["round"] == 5) - adapter.close() - assert renders == ["a"] - - def test_plugin_write_keeps_the_round_recorded_by_the_live_snapshot( tmp_path: Path, renders: list[str], fake_search_config: None ) -> None: @@ -765,14 +766,6 @@ def _subscribed(tmp_path: Path) -> Any: return callbacks -def test_subscribe_registers_the_lifecycle_callbacks( - tmp_path: Path, renders: list[str] -) -> None: - callbacks = _subscribed(tmp_path) - assert callbacks.on_complete is not None - assert callbacks.on_failure is not None - - def test_a_failed_search_leaves_a_terminal_failed_snapshot( tmp_path: Path, renders: list[str] ) -> None: From 69497493204a1a1f37b1da4b38edac52456460b3 Mon Sep 17 00:00:00 2001 From: devivasudevan <49675305+devivasudevan@users.noreply.github.com> Date: Thu, 8 Oct 2026 15:33:14 +0200 Subject: [PATCH 06/40] fix(operator): support multi-document candidate manifests, recreate incomplete candidates --- .../operator/internal/dgdcreconcile/diff.go | 1 - .../internal/dgdrrunpublisher/kube.go | 89 ++++++++++++++----- .../internal/dgdrrunpublisher/publisher.go | 1 - .../dgdrrunpublisher/publisher_test.go | 51 +++++++---- .../internal/dgdrrunpublisher/snapshot.go | 54 +++++++++-- 5 files changed, 147 insertions(+), 49 deletions(-) diff --git a/deploy/operator/internal/dgdcreconcile/diff.go b/deploy/operator/internal/dgdcreconcile/diff.go index 26e48c2e23ed..fcf36906c89d 100644 --- a/deploy/operator/internal/dgdcreconcile/diff.go +++ b/deploy/operator/internal/dgdcreconcile/diff.go @@ -44,7 +44,6 @@ func newDiffInputError(format string, args ...any) *DiffInputError { return &DiffInputError{msg: fmt.Sprintf(format, args...)} } -// DesiredCandidate is one candidate the snapshot wants to exist. type DesiredCandidate struct { // ID is the stable candidate identity from the snapshot. ID string diff --git a/deploy/operator/internal/dgdrrunpublisher/kube.go b/deploy/operator/internal/dgdrrunpublisher/kube.go index 31355f781ad1..62e1a21fe04a 100644 --- a/deploy/operator/internal/dgdrrunpublisher/kube.go +++ b/deploy/operator/internal/dgdrrunpublisher/kube.go @@ -21,7 +21,9 @@ import ( "context" "encoding/json" "fmt" + "strings" + dgdv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1" v1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" "github.com/ai-dynamo/dynamo/deploy/operator/internal/dgdcreconcile" v1beta2 "github.com/ai-dynamo/dynamo/deploy/operator/internal/dgdrrunpublisher/placeholderapi" @@ -29,11 +31,11 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" - "sigs.k8s.io/yaml" ) const ( @@ -45,7 +47,11 @@ const ( LabelCandidateID = "nvidia.com/dgdr-candidate-id" evaluatedCondition = "Evaluated" - dgdKind = "DynamoGraphDeployment" + + // AnnotationAdditionalResources carries the companion resources (for example + // generated ConfigMaps) rendered next to the DGD, as multi-document YAML. It is the + // same key and format the DGDR controller uses. + AnnotationAdditionalResources = "dgdr.nvidia.com/additional-resources" ) // KubeCluster is the Cluster implementation backed by a controller-runtime client. It @@ -94,30 +100,40 @@ func (k *KubeCluster) ListCandidates(ctx context.Context) ([]dgdcreconcile.Curre return out, nil } -// decodeSpec extracts the DGD spec from a rendered manifest, rejecting anything that is -// not a DynamoGraphDeployment with a spec rather than creating an empty candidate. -func decodeSpec(candidate dgdcreconcile.DesiredCandidate) (*v1beta1.DynamoGraphDeploymentSpec, error) { - var doc struct { - Kind string `json:"kind"` - Spec *v1beta1.DynamoGraphDeploymentSpec `json:"spec"` - } - if err := yaml.Unmarshal([]byte(candidate.Spec), &doc); err != nil { - return nil, fmt.Errorf("decoding manifest of candidate %s: %w", candidate.ID, err) - } - if doc.Kind != dgdKind { - return nil, fmt.Errorf("manifest of candidate %s is a %q, want %s", candidate.ID, doc.Kind, dgdKind) +// decodeManifest extracts the DGD spec and the companion resources from a rendered +// manifest. A DGD in either supported apiVersion is converted to v1beta1, the same way +// the DGDR controller does it. +func decodeManifest(candidate dgdcreconcile.DesiredCandidate) (*v1beta1.DynamoGraphDeploymentSpec, string, error) { + manifest, err := ParseManifest(candidate.Spec) + if err != nil { + return nil, "", fmt.Errorf("candidate %s: %w", candidate.ID, err) } - if doc.Spec == nil { - return nil, fmt.Errorf("manifest of candidate %s has no spec", candidate.ID) + obj := &unstructured.Unstructured{Object: manifest.DGD} + var beta v1beta1.DynamoGraphDeployment + switch obj.GetAPIVersion() { + case dgdv1alpha1.GroupVersion.String(): + var alpha dgdv1alpha1.DynamoGraphDeployment + if err := runtime.DefaultUnstructuredConverter.FromUnstructured(obj.Object, &alpha); err != nil { + return nil, "", fmt.Errorf("candidate %s: %w", candidate.ID, err) + } + if err := alpha.ConvertTo(&beta); err != nil { + return nil, "", fmt.Errorf("candidate %s: %w", candidate.ID, err) + } + case v1beta1.GroupVersion.String(): + if err := runtime.DefaultUnstructuredConverter.FromUnstructured(obj.Object, &beta); err != nil { + return nil, "", fmt.Errorf("candidate %s: %w", candidate.ID, err) + } + default: + return nil, "", fmt.Errorf("candidate %s: unsupported DynamoGraphDeployment apiVersion %q", candidate.ID, obj.GetAPIVersion()) } - return doc.Spec, nil + return &beta.Spec, strings.Join(manifest.Companions, "\n---\n"), nil } // CreateCandidate creates the candidate and populates its status exactly once. Both // steps are idempotent so a retry after a partial failure converges; the publisher // calls it again for a candidate whose status was never populated. func (k *KubeCluster) CreateCandidate(ctx context.Context, name string, candidate dgdcreconcile.DesiredCandidate) error { - spec, err := decodeSpec(candidate) + spec, companions, err := decodeManifest(candidate) if err != nil { return err } @@ -140,10 +156,17 @@ func (k *KubeCluster) CreateCandidate(ctx context.Context, name string, candidat Parameters: parameters, }, } + if companions != "" { + object.Annotations = map[string]string{AnnotationAdditionalResources: companions} + } if err := controllerutil.SetControllerReference(run, object, k.Client.Scheme()); err != nil { return err } - if err := k.Client.Create(ctx, object); err != nil && !apierrors.IsAlreadyExists(err) { + err = k.Client.Create(ctx, object) + if apierrors.IsAlreadyExists(err) { + err = k.replaceIfIncomplete(ctx, object, run) + } + if err != nil { return err } @@ -151,9 +174,6 @@ func (k *KubeCluster) CreateCandidate(ctx context.Context, name string, candidat if err := k.Client.Get(ctx, types.NamespacedName{Namespace: k.Namespace, Name: name}, &stored); err != nil { return err } - if !metav1.IsControlledBy(&stored, run) { - return fmt.Errorf("candidate %s already exists and is not controlled by run %s (uid %s)", name, k.RunName, run.UID) - } if meta.IsStatusConditionTrue(stored.Status.Conditions, evaluatedCondition) { return nil // status already populated; candidates are immutable afterwards } @@ -171,6 +191,31 @@ func (k *KubeCluster) CreateCandidate(ctx context.Context, name string, candidat return k.Client.Status().Update(ctx, &stored) } +// replaceIfIncomplete handles a candidate that already exists. One controlled by another +// run is an error. A complete one is left untouched (candidates are immutable). One whose +// status was never written may hold the spec of an earlier snapshot that reused the id, +// so it is deleted and created again from the current manifest. +func (k *KubeCluster) replaceIfIncomplete(ctx context.Context, object *v1beta2.DynamoGraphDeploymentCandidate, run *v1beta2.DynamoGraphDeploymentRun) error { + var stored v1beta2.DynamoGraphDeploymentCandidate + if err := k.Client.Get(ctx, types.NamespacedName{Namespace: k.Namespace, Name: object.Name}, &stored); err != nil { + return err + } + if !metav1.IsControlledBy(&stored, run) { + return fmt.Errorf("candidate %s already exists and is not controlled by run %s (uid %s)", object.Name, k.RunName, run.UID) + } + if meta.IsStatusConditionTrue(stored.Status.Conditions, evaluatedCondition) { + return nil + } + uid := stored.UID + if err := k.Client.Delete(ctx, &stored, client.Preconditions{UID: &uid}); client.IgnoreNotFound(err) != nil { + return err + } + if err := k.Client.Create(ctx, object.DeepCopy()); err != nil { + return fmt.Errorf("recreating incomplete candidate %s: %w", object.Name, err) + } + return nil +} + func rawExtension(value map[string]any) (*runtime.RawExtension, error) { if value == nil { value = map[string]any{} diff --git a/deploy/operator/internal/dgdrrunpublisher/publisher.go b/deploy/operator/internal/dgdrrunpublisher/publisher.go index 213f5cb15551..0f3c011e9468 100644 --- a/deploy/operator/internal/dgdrrunpublisher/publisher.go +++ b/deploy/operator/internal/dgdrrunpublisher/publisher.go @@ -118,7 +118,6 @@ type Cluster interface { SweeperState(ctx context.Context) (SweeperState, error) } -// Publisher reconciles snapshots into the cluster. type Publisher struct { Cluster Cluster SnapshotDir string diff --git a/deploy/operator/internal/dgdrrunpublisher/publisher_test.go b/deploy/operator/internal/dgdrrunpublisher/publisher_test.go index e1d62c901c8c..81c23847710a 100644 --- a/deploy/operator/internal/dgdrrunpublisher/publisher_test.go +++ b/deploy/operator/internal/dgdrrunpublisher/publisher_test.go @@ -402,23 +402,14 @@ func TestRunTerminalSnapshotAcknowledges(t *testing.T) { } func TestRunTerminalFailedSnapshotIsReconciledThenFails(t *testing.T) { - for name, state := range map[string]SweeperState{ - "sweeper exit 0": {Exited: true, ExitCode: 0}, - "sweeper exit 1": {Exited: true, ExitCode: 1}, - "still running": {}, - } { - t.Run(name, func(t *testing.T) { - p, fc, dir := newPublisher(t) - writeSnapshot(t, dir, snapshotJSON(t, PhaseFailed, cand{id: "a"})) - fc.setState(state) - err := runWithTimeout(t, p) - if !errors.Is(err, ErrRunFailed) || ExitCode(err) != ExitRunFailed { - t.Fatalf("err = %v", err) - } - if !reflect.DeepEqual(fc.creates, []string{"run-a"}) { - t.Fatalf("creates = %v", fc.creates) - } - }) + p, fc, dir := newPublisher(t) + writeSnapshot(t, dir, snapshotJSON(t, PhaseFailed, cand{id: "a"})) + err := runWithTimeout(t, p) + if !errors.Is(err, ErrRunFailed) || ExitCode(err) != ExitRunFailed { + t.Fatalf("err = %v", err) + } + if !reflect.DeepEqual(fc.creates, []string{"run-a"}) { + t.Fatalf("creates = %v", fc.creates) } } @@ -640,3 +631,29 @@ func TestInvalidFinalSnapshotIsAProtocolViolationExit(t *testing.T) { t.Fatalf("err = %v, exit = %d", err, ExitCode(err)) } } + +func TestParseManifestHandlesCompanionResources(t *testing.T) { + dgd := "apiVersion: nvidia.com/v1beta1\nkind: DynamoGraphDeployment\nspec: {}\n" + cm := "apiVersion: v1\nkind: ConfigMap\nmetadata: {}\n" + for name, manifest := range map[string]string{ + "dgd first": dgd + "---\n" + cm, + "configmap first": "# generated\n---\n" + cm + "---\n" + dgd, + } { + got, err := ParseManifest(manifest) + if err != nil { + t.Fatalf("%s: %v", name, err) + } + if got.DGD["kind"] != "DynamoGraphDeployment" || len(got.Companions) != 1 || !strings.Contains(got.Companions[0], "ConfigMap") { + t.Fatalf("%s: %+v", name, got) + } + } + for name, manifest := range map[string]string{ + "two dgds": dgd + "---\n" + dgd, + "no dgd": cm, + "no spec": "kind: DynamoGraphDeployment\n", + } { + if _, err := ParseManifest(manifest); err == nil { + t.Errorf("%s: want error", name) + } + } +} diff --git a/deploy/operator/internal/dgdrrunpublisher/snapshot.go b/deploy/operator/internal/dgdrrunpublisher/snapshot.go index ef5ad6f2b9ce..a5c4224ea395 100644 --- a/deploy/operator/internal/dgdrrunpublisher/snapshot.go +++ b/deploy/operator/internal/dgdrrunpublisher/snapshot.go @@ -21,6 +21,7 @@ import ( "errors" "fmt" "regexp" + "strings" "sigs.k8s.io/yaml" ) @@ -42,16 +43,53 @@ const maxCandidateIDLength = 48 // object name. var candidateIDPattern = regexp.MustCompile(`^[a-z0-9]([-a-z0-9]*[a-z0-9])?$`) -func validateManifest(id, manifest string) error { - var doc map[string]any - if err := yaml.Unmarshal([]byte(manifest), &doc); err != nil { - return violation("manifest of candidate %q is not a YAML mapping: %v", id, err) +const dgdKind = "DynamoGraphDeployment" + +// Manifest is one rendered candidate: exactly one DynamoGraphDeployment plus any +// companion resources (for example generated ConfigMaps) the renderer emitted with it. +type Manifest struct { + DGD map[string]any + Companions []string // raw YAML documents, in order +} + +var documentSeparator = regexp.MustCompile(`(?m)^---[ \t]*$`) + +// ParseManifest splits a multi-document manifest and locates its single +// DynamoGraphDeployment, which must carry a spec. +func ParseManifest(manifest string) (*Manifest, error) { + var out Manifest + for _, doc := range documentSeparator.Split(manifest, -1) { + if strings.TrimSpace(doc) == "" { + continue + } + var obj map[string]any + if err := yaml.Unmarshal([]byte(doc), &obj); err != nil { + return nil, fmt.Errorf("manifest document is not a YAML mapping: %w", err) + } + if len(obj) == 0 { + continue // comment-only document + } + if obj["kind"] != dgdKind { + out.Companions = append(out.Companions, strings.TrimSpace(doc)) + continue + } + if out.DGD != nil { + return nil, errors.New("manifest has more than one DynamoGraphDeployment") + } + if _, ok := obj["spec"].(map[string]any); !ok { + return nil, errors.New("DynamoGraphDeployment has no spec") + } + out.DGD = obj } - if doc["kind"] != "DynamoGraphDeployment" { - return violation("manifest of candidate %q is not a DynamoGraphDeployment", id) + if out.DGD == nil { + return nil, errors.New("manifest has no DynamoGraphDeployment") } - if _, ok := doc["spec"].(map[string]any); !ok { - return violation("manifest of candidate %q has no spec", id) + return &out, nil +} + +func validateManifest(id, manifest string) error { + if _, err := ParseManifest(manifest); err != nil { + return violation("manifest of candidate %q: %v", id, err) } return nil } From 314b9c8f3d230a5bdfe71353c95afc069d258d99 Mon Sep 17 00:00:00 2001 From: devivasudevan <49675305+devivasudevan@users.noreply.github.com> Date: Thu, 8 Oct 2026 15:39:13 +0200 Subject: [PATCH 07/40] fix(aisimulate): terminal failure on broken renderer, JSON-safe metrics, bounded errors, require lifecycle hooks --- .../aisimulate/output/dgdr_run/adapter.py | 89 ++++++++++++++++--- .../aisimulate/tests/dgdr_run/test_adapter.py | 82 ++++++++++++++--- 2 files changed, 145 insertions(+), 26 deletions(-) diff --git a/components/src/dynamo/aisimulate/output/dgdr_run/adapter.py b/components/src/dynamo/aisimulate/output/dgdr_run/adapter.py index a3cf32f8b283..790e6e755b17 100644 --- a/components/src/dynamo/aisimulate/output/dgdr_run/adapter.py +++ b/components/src/dynamo/aisimulate/output/dgdr_run/adapter.py @@ -44,6 +44,7 @@ import hashlib import json import logging +import math import threading import time from collections.abc import Mapping, Sequence @@ -97,6 +98,25 @@ def _plain(value: Any) -> Any: return json.loads(json.dumps(value, default=str, sort_keys=True)) +def _bounded(text: str) -> str: + return text[:_MAX_ERROR_MESSAGE] + + +def _finite(value: Any) -> Any: + """Replace non-finite floats with ``None`` so the snapshot stays valid JSON/YAML.""" + if isinstance(value, float): + return value if math.isfinite(value) else None + if isinstance(value, dict): + return {key: _finite(item) for key, item in value.items()} + if isinstance(value, list): + return [_finite(item) for item in value] + return value + + +class _RendererUnavailable(Exception): + """Rendering failed for a reason that is not specific to one candidate.""" + + def candidate_id_for(parameters: Mapping[str, Any]) -> str: """Stable identity of an evaluated point. @@ -290,16 +310,18 @@ def _retain(record: CandidateRecordLike) -> _Retained | None: ) return None parameters = _plain(dict(record.config)) + score = float(record.score) metrics: dict[str, Any] = { - "score": float(record.score), + "score": score if math.isfinite(score) else None, "usedGpus": int(record.used_gpus), } objectives = getattr(record, "objectives", None) if objectives: - metrics["objectives"] = _plain(dict(objectives)) + metrics["objectives"] = _finite(_plain(dict(objectives))) return _Retained( id=candidate_id_for(parameters), - score=float(record.score), + # A NaN score has no place in an ordering; it ranks as the worst. + score=-math.inf if math.isnan(score) else score, used_gpus=int(record.used_gpus), parameters=parameters, metrics=metrics, @@ -343,7 +365,30 @@ def close( self._wake.set() if self._thread is not None: self._thread.join() - self._publish() + self._publish_or_record_failure() + + def _publish_or_record_failure(self) -> None: + """Publish; if rendering itself is broken, record that as a terminal failure. + + A broken renderer is a run-level problem, not a candidate-specific one, so it + must not become ``MATERIALIZATION_FAILED``. The terminal ``Failed`` snapshot + carries the candidates that were already rendered, and the original exception + is re-raised. Failures to write the file keep propagating unchanged. + """ + try: + self._publish() + except _RendererUnavailable as unavailable: + cause = unavailable.__cause__ or unavailable + logger.error("dgdr_run: renderer unavailable", exc_info=cause) + with self._lock: + self._accepting = False + self._terminal = _Terminal( + phase=RunPhase.FAILED, + message=_bounded(str(cause)), + error=type(cause).__name__, + ) + self._publish(render=False) + raise cause def finish_search(self) -> None: """The search ended normally: flush the last live state and stop the writer. @@ -364,14 +409,14 @@ def finish_search(self) -> None: self._wake.set() if self._thread is not None: self._thread.join() - self._publish() + self._publish_or_record_failure() def fail(self, error: BaseException) -> None: """The search raised: write the terminal ``Failed`` snapshot.""" self.close( phase=RunPhase.FAILED, error=type(error).__name__, - message=str(error)[:_MAX_ERROR_MESSAGE], + message=_bounded(str(error)), ) def __enter__(self) -> DGDRRunOutputAdapter: @@ -404,7 +449,7 @@ def _writer_loop(self) -> None: self._wake.set() self._next_allowed = time.monotonic() + self._interval - def _publish(self) -> None: + def _publish(self, *, render: bool = True) -> None: with self._publish_lock: with self._lock: # Cleared together with the copy so no update is ever lost: an @@ -418,7 +463,14 @@ def _publish(self) -> None: round_no = self._round_no evaluated = self._evaluated terminal = self._terminal - candidates = tuple(self._materialize(item) for item in ordered) + if render: + candidates = tuple(self._materialize(item) for item in ordered) + else: + candidates = tuple( + self._materialized[item.id] + for item in ordered + if item.id in self._materialized + ) live_ids = {item.id for item in ordered} for stale in set(self._materialized) - live_ids: del self._materialized[stale] @@ -465,8 +517,10 @@ def _materialize(self, item: _Retained) -> SnapshotCandidate: outcome=CandidateOutcome.MATERIALIZATION_FAILED, parameters=item.parameters, metrics=item.metrics, - error=str(exc), + error=_bounded(str(exc)), ) + except Exception as exc: + raise _RendererUnavailable(str(exc)) from exc self._materialized[item.id] = entry return entry @@ -509,16 +563,23 @@ def subscribe( workload = getattr(context, "workload", None) if workload is None or resolved.snapshot_dir is None: return None + # Live publishing needs both lifecycle hooks: without on_complete nothing stops + # the live writer before write() publishes the terminal snapshot, which a pending + # live write could then overwrite. Older AISimulate releases lack them; the + # terminal snapshot then comes from write() alone. + supported = {field.name for field in fields(RecommendationOutputCallbacks)} + if not {"on_complete", "on_failure"} <= supported: + logger.info( + "dgdr_run: AISimulate has no lifecycle callbacks; not publishing live" + ) + return None adapter = DGDRRunOutputAdapter(resolved, workload) adapter.start() - lifecycle = {"on_complete": adapter.finish_search, "on_failure": adapter.fail} - # Older AISimulate releases have no lifecycle callbacks; the terminal snapshot - # then comes from write() alone and a failed run is detected by the publisher. - supported = {field.name for field in fields(RecommendationOutputCallbacks)} return RecommendationOutputCallbacks( on_candidate=adapter.on_candidate, on_round=adapter.on_round, - **{name: hook for name, hook in lifecycle.items() if name in supported}, + on_complete=adapter.finish_search, + on_failure=adapter.fail, ) def live( diff --git a/components/src/dynamo/aisimulate/tests/dgdr_run/test_adapter.py b/components/src/dynamo/aisimulate/tests/dgdr_run/test_adapter.py index f076253a24fc..3ad6a2df075f 100644 --- a/components/src/dynamo/aisimulate/tests/dgdr_run/test_adapter.py +++ b/components/src/dynamo/aisimulate/tests/dgdr_run/test_adapter.py @@ -636,6 +636,8 @@ def test_plugin_subscribe_with_a_context_publishes_live( callbacks.on_candidate(record("a", 9.0)) wait_for(lambda: tags(load(tmp_path)) == ["a"]) + assert callbacks.on_complete is not None + callbacks.on_complete() def test_plugin_subscribe_without_context_or_snapshot_dir_only_validates( @@ -671,13 +673,6 @@ def model_validate(_: Any) -> Any: monkeypatch.setattr(adapter_module, "SmartSearchConfig", FakeSearchConfig) -def test_plugin_write_records_progress_from_the_result_counts( - tmp_path: Path, renders: list[str], fake_search_config: None -) -> None: - create_adapter().write(PLUGIN_CONFIG, result=_final_result(42), output_dir=tmp_path) - assert load(tmp_path)["progress"]["evaluated"] == 42 - - def test_plugin_write_reports_no_artifact_for_a_snapshot_outside_output_dir( tmp_path: Path, renders: list[str], fake_search_config: None ) -> None: @@ -735,6 +730,8 @@ def test_plugin_write_keeps_the_round_recorded_by_the_live_snapshot( assert live is not None and live.on_round is not None live.on_round(3, []) wait_for(lambda: load(tmp_path)["progress"]["round"] == 3) + assert live.on_complete is not None + live.on_complete() create_adapter().write( {**PLUGIN_CONFIG, "snapshot_dir": str(tmp_path)}, @@ -861,10 +858,68 @@ def test_failure_after_completion_is_a_noop_and_completion_after_failure_too( assert load(tmp_path)["run"]["message"] == "first" -def test_subscribe_still_works_with_an_aisimulate_without_lifecycle_callbacks( +# -- robustness ----------------------------------------------------------------- + + +def test_a_broken_renderer_leaves_a_terminal_failed_snapshot( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + def broken(*_: Any, **__: Any) -> str: + raise RuntimeError("renderer not installed") + + monkeypatch.setattr(adapter_module, "render_dgd", broken) + adapter = DGDRRunOutputAdapter(make_config(tmp_path), workload=None) + adapter.on_candidate(record("a", 9.0)) + + with pytest.raises(RuntimeError, match="renderer not installed"): + adapter.close() + + snapshot = load(tmp_path) + assert snapshot["run"]["phase"] == "Failed" + assert snapshot["run"]["terminal"] is True + assert snapshot["run"]["error"] == "RuntimeError" + assert all(c["outcome"] != "materialization_failed" for c in snapshot["candidates"]) + + +def test_non_finite_scores_and_objectives_stay_valid_for_the_publisher( + tmp_path: Path, renders: list[str] +) -> None: + adapter = DGDRRunOutputAdapter(make_config(tmp_path), workload=None) + adapter.on_candidate(record("a", float("-inf"), objectives={"e2e": float("nan")})) + adapter.on_candidate(record("b", float("nan"))) + adapter.close() + + text = (tmp_path / SNAPSHOT_FILE_NAME).read_text() + assert ( + "inf" not in text.lower().replace("information", "") + and "nan" not in text.lower() + ) + metrics = { + c["parameters"]["tag"]: c["metrics"] for c in load(tmp_path)["candidates"] + } + assert metrics["a"]["score"] is None and metrics["a"]["objectives"] == {"e2e": None} + assert metrics["b"]["score"] is None + + +def test_a_candidate_error_message_is_bounded( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + def failing(*_: Any, **__: Any) -> str: + raise CandidateMaterializationError("x" * 5000) + + monkeypatch.setattr(adapter_module, "render_dgd", failing) + adapter = DGDRRunOutputAdapter(make_config(tmp_path), workload=None) + adapter.on_candidate(record("a", 9.0)) + adapter.close() + + (candidate,) = load(tmp_path)["candidates"] + assert 0 < len(candidate["error"]) <= 500 + + +def test_subscribe_does_not_publish_live_without_lifecycle_callbacks( tmp_path: Path, renders: list[str], monkeypatch: pytest.MonkeyPatch ) -> None: - @dataclass(frozen=True) + @dataclass class LegacyCallbacks: on_candidate: Any = None on_round: Any = None @@ -872,6 +927,9 @@ class LegacyCallbacks: monkeypatch.setattr( adapter_module, "RecommendationOutputCallbacks", LegacyCallbacks ) - callbacks = _subscribed(tmp_path) - assert isinstance(callbacks, LegacyCallbacks) - assert callbacks.on_candidate is not None + callbacks = create_adapter().subscribe( + {**PLUGIN_CONFIG, "snapshot_dir": str(tmp_path)}, + context=SimpleNamespace(workload="workload"), + ) + assert callbacks is None + assert not (tmp_path / SNAPSHOT_FILE_NAME).exists() From 84f64737796018276b19f2a6a987b025550201bc Mon Sep 17 00:00:00 2001 From: devivasudevan <49675305+devivasudevan@users.noreply.github.com> Date: Thu, 8 Oct 2026 15:57:16 +0200 Subject: [PATCH 08/40] fix(operator): validate DGD apiVersion in snapshots; trim redundant comments and tests --- .../dgdrrunpublisher/placeholderapi/types.go | 2 -- .../dgdrrunpublisher/publisher_test.go | 19 ++++++++++--------- .../internal/dgdrrunpublisher/snapshot.go | 9 +++++++++ 3 files changed, 19 insertions(+), 11 deletions(-) diff --git a/deploy/operator/internal/dgdrrunpublisher/placeholderapi/types.go b/deploy/operator/internal/dgdrrunpublisher/placeholderapi/types.go index 8541668797d7..a743ed40a615 100644 --- a/deploy/operator/internal/dgdrrunpublisher/placeholderapi/types.go +++ b/deploy/operator/internal/dgdrrunpublisher/placeholderapi/types.go @@ -83,7 +83,6 @@ type DynamoGraphDeploymentCandidate struct { Status DynamoGraphDeploymentCandidateStatus `json:"status,omitempty"` } -// DynamoGraphDeploymentCandidateList contains a list of candidates. type DynamoGraphDeploymentCandidateList struct { metav1.TypeMeta `json:",inline"` metav1.ListMeta `json:"metadata,omitempty"` @@ -140,7 +139,6 @@ type DynamoGraphDeploymentRun struct { Status DynamoGraphDeploymentRunStatus `json:"status,omitempty"` } -// DynamoGraphDeploymentRunList contains a list of runs. type DynamoGraphDeploymentRunList struct { metav1.TypeMeta `json:",inline"` metav1.ListMeta `json:"metadata,omitempty"` diff --git a/deploy/operator/internal/dgdrrunpublisher/publisher_test.go b/deploy/operator/internal/dgdrrunpublisher/publisher_test.go index 81c23847710a..35814c6f8ed2 100644 --- a/deploy/operator/internal/dgdrrunpublisher/publisher_test.go +++ b/deploy/operator/internal/dgdrrunpublisher/publisher_test.go @@ -109,7 +109,7 @@ func (f *fakeCluster) setState(s SweeperState) { f.state = s } -const validManifest = "kind: DynamoGraphDeployment\nspec: {}\n" +const validManifest = "apiVersion: nvidia.com/v1beta1\nkind: DynamoGraphDeployment\nspec: {}\n" type cand struct { id string @@ -355,14 +355,13 @@ func TestParseSnapshotValidation(t *testing.T) { "schema": `{"schemaVersion":2,"run":{"phase":"Running","terminal":false}}`, "phase": `{"schemaVersion":1,"run":{"phase":"Weird","terminal":false}}`, "terminal flag": `{"schemaVersion":1,"run":{"phase":"Succeeded","terminal":false}}`, - "duplicate": `{"schemaVersion":1,"run":{"phase":"Running","terminal":false},"candidates":[{"id":"a","outcome":"materialized","manifest":"kind: DynamoGraphDeployment\\nspec: {}\\n"},{"id":"a","outcome":"materialized","manifest":"kind: DynamoGraphDeployment\\nspec: {}\\n"}]}`, + "duplicate": `{"schemaVersion":1,"run":{"phase":"Running","terminal":false},"candidates":[{"id":"a","outcome":"materialized","manifest":"apiVersion: nvidia.com/v1beta1\\nkind: DynamoGraphDeployment\\nspec: {}\\n"},{"id":"a","outcome":"materialized","manifest":"apiVersion: nvidia.com/v1beta1\\nkind: DynamoGraphDeployment\\nspec: {}\\n"}]}`, "no manifest": `{"schemaVersion":1,"run":{"phase":"Running","terminal":false},"candidates":[{"id":"a","outcome":"materialized"}]}`, "no error": `{"schemaVersion":1,"run":{"phase":"Running","terminal":false},"candidates":[{"id":"a","outcome":"materialization_failed"}]}`, - "unsafe id": `{"schemaVersion":1,"run":{"phase":"Running","terminal":false},"candidates":[{"id":"Not_Valid","outcome":"materialized","manifest":"kind: DynamoGraphDeployment\\nspec: {}\\n"}]}`, - "long id": `{"schemaVersion":1,"run":{"phase":"Running","terminal":false},"candidates":[{"id":"%s","outcome":"materialized","manifest":"kind: DynamoGraphDeployment\\nspec: {}\\n"}]}`, + "unsafe id": `{"schemaVersion":1,"run":{"phase":"Running","terminal":false},"candidates":[{"id":"Not_Valid","outcome":"materialized","manifest":"apiVersion: nvidia.com/v1beta1\\nkind: DynamoGraphDeployment\\nspec: {}\\n"}]}`, + "long id": `{"schemaVersion":1,"run":{"phase":"Running","terminal":false},"candidates":[{"id":"%s","outcome":"materialized","manifest":"apiVersion: nvidia.com/v1beta1\\nkind: DynamoGraphDeployment\\nspec: {}\\n"}]}`, "wrong kind": `{"schemaVersion":1,"run":{"phase":"Running","terminal":false},"candidates":[{"id":"a","outcome":"materialized","manifest":"kind: ConfigMap\\nspec: {}\\n"}]}`, - "no spec": `{"schemaVersion":1,"run":{"phase":"Running","terminal":false},"candidates":[{"id":"a","outcome":"materialized","manifest":"kind: DynamoGraphDeployment\\n"}]}`, - "empty id": `{"schemaVersion":1,"run":{"phase":"Running","terminal":false},"candidates":[{"id":"","outcome":"materialized","manifest":"kind: DynamoGraphDeployment\\nspec: {}\\n"}]}`, + "empty id": `{"schemaVersion":1,"run":{"phase":"Running","terminal":false},"candidates":[{"id":"","outcome":"materialized","manifest":"apiVersion: nvidia.com/v1beta1\\nkind: DynamoGraphDeployment\\nspec: {}\\n"}]}`, } bad["long id"] = strings.Replace(bad["long id"], "%s", strings.Repeat("a", 64), 1) for name, doc := range bad { @@ -648,9 +647,11 @@ func TestParseManifestHandlesCompanionResources(t *testing.T) { } } for name, manifest := range map[string]string{ - "two dgds": dgd + "---\n" + dgd, - "no dgd": cm, - "no spec": "kind: DynamoGraphDeployment\n", + "two dgds": dgd + "---\n" + dgd, + "no dgd": cm, + "no spec": "apiVersion: nvidia.com/v1beta1\nkind: DynamoGraphDeployment\n", + "no apiVersion": "kind: DynamoGraphDeployment\nspec: {}\n", + "unsupported apiVersion": "apiVersion: nvidia.com/v9\nkind: DynamoGraphDeployment\nspec: {}\n", } { if _, err := ParseManifest(manifest); err == nil { t.Errorf("%s: want error", name) diff --git a/deploy/operator/internal/dgdrrunpublisher/snapshot.go b/deploy/operator/internal/dgdrrunpublisher/snapshot.go index a5c4224ea395..5bbe05125bbd 100644 --- a/deploy/operator/internal/dgdrrunpublisher/snapshot.go +++ b/deploy/operator/internal/dgdrrunpublisher/snapshot.go @@ -45,6 +45,12 @@ var candidateIDPattern = regexp.MustCompile(`^[a-z0-9]([-a-z0-9]*[a-z0-9])?$`) const dgdKind = "DynamoGraphDeployment" +// supportedDGDAPIVersions are the DGD versions the candidate path can convert to v1beta1. +var supportedDGDAPIVersions = map[string]bool{ + "nvidia.com/v1alpha1": true, + "nvidia.com/v1beta1": true, +} + // Manifest is one rendered candidate: exactly one DynamoGraphDeployment plus any // companion resources (for example generated ConfigMaps) the renderer emitted with it. type Manifest struct { @@ -76,6 +82,9 @@ func ParseManifest(manifest string) (*Manifest, error) { if out.DGD != nil { return nil, errors.New("manifest has more than one DynamoGraphDeployment") } + if version, _ := obj["apiVersion"].(string); !supportedDGDAPIVersions[version] { + return nil, fmt.Errorf("DynamoGraphDeployment has unsupported apiVersion %q", version) + } if _, ok := obj["spec"].(map[string]any); !ok { return nil, errors.New("DynamoGraphDeployment has no spec") } From c120ddf08cc648978526ef7c1437111e29341835 Mon Sep 17 00:00:00 2001 From: devivasudevan <49675305+devivasudevan@users.noreply.github.com> Date: Thu, 8 Oct 2026 16:19:58 +0200 Subject: [PATCH 09/40] fix(aisimulate): back off failed snapshot writes, keep objectives in terminal snapshot; trim tests --- .../aisimulate/output/dgdr_run/adapter.py | 8 ++- .../aisimulate/tests/dgdr_run/test_adapter.py | 58 ++++++++++--------- 2 files changed, 39 insertions(+), 27 deletions(-) diff --git a/components/src/dynamo/aisimulate/output/dgdr_run/adapter.py b/components/src/dynamo/aisimulate/output/dgdr_run/adapter.py index 790e6e755b17..2f493a5c075f 100644 --- a/components/src/dynamo/aisimulate/output/dgdr_run/adapter.py +++ b/components/src/dynamo/aisimulate/output/dgdr_run/adapter.py @@ -80,6 +80,7 @@ _ID_PREFIX = "evaluated-point-" _MAX_RENDER_ATTEMPTS = 3 _MAX_ERROR_MESSAGE = 500 +_WRITE_RETRY_BACKOFF = 1.0 _ID_HEX_LENGTH = 12 @@ -442,12 +443,15 @@ def _writer_loop(self) -> None: delay = self._next_allowed - time.monotonic() if delay > 0 and self._stop.wait(delay): return + wait = self._interval try: self._publish() except Exception: # keep the writer alive; retry logger.exception("dgdr_run: snapshot write failed; will retry") self._wake.set() - self._next_allowed = time.monotonic() + self._interval + # Even with a zero interval, a persistent failure must not spin. + wait = max(wait, _WRITE_RETRY_BACKOFF) + self._next_allowed = time.monotonic() + wait def _publish(self, *, render: bool = True) -> None: with self._publish_lock: @@ -644,6 +648,7 @@ class _FinalCandidate: config: dict[str, Any] score: float | None used_gpus: int | None + objectives: dict[str, float] | None = None status: str = "feasible" @@ -659,6 +664,7 @@ def _as_feasible(candidate: Any) -> _FinalCandidate: config=dict(candidate.config), score=score, used_gpus=used_gpus, + objectives=getattr(candidate, "objectives", None), ) diff --git a/components/src/dynamo/aisimulate/tests/dgdr_run/test_adapter.py b/components/src/dynamo/aisimulate/tests/dgdr_run/test_adapter.py index 3ad6a2df075f..4060646bea09 100644 --- a/components/src/dynamo/aisimulate/tests/dgdr_run/test_adapter.py +++ b/components/src/dynamo/aisimulate/tests/dgdr_run/test_adapter.py @@ -201,7 +201,6 @@ def test_identity_and_manifests_do_not_depend_on_rank_or_observation_order( one.on_candidate(record("a", 1.0)) one.on_candidate(record("b", 2.0)) one.close() - # Same two points, opposite scores and opposite observation order. two = DGDRRunOutputAdapter(make_config(second), workload=None) two.on_candidate(record("b", 1.0)) two.on_candidate(record("a", 2.0)) @@ -443,6 +442,7 @@ def broken(*_: Any, **__: Any) -> None: def test_a_failed_periodic_write_is_retried_with_the_latest_state( tmp_path: Path, renders: list[str], monkeypatch: pytest.MonkeyPatch ) -> None: + monkeypatch.setattr(adapter_module, "_WRITE_RETRY_BACKOFF", 0.02) adapter = DGDRRunOutputAdapter(make_config(tmp_path), workload=None) adapter.start() real = adapter_module.write_snapshot @@ -631,7 +631,6 @@ def test_plugin_subscribe_with_a_context_publishes_live( assert callbacks is not None assert callbacks.on_candidate is not None assert callbacks.on_round is not None - # The initial snapshot exists before the first candidate is evaluated. assert load(tmp_path)["run"] == {"phase": "Running", "terminal": False} callbacks.on_candidate(record("a", 9.0)) @@ -720,30 +719,6 @@ def flaky_render(candidate: Any, workload: Any, options: Any, **kwargs: Any) -> assert len(attempts) == 2 -def test_plugin_write_keeps_the_round_recorded_by_the_live_snapshot( - tmp_path: Path, renders: list[str], fake_search_config: None -) -> None: - live = create_adapter().subscribe( - {**PLUGIN_CONFIG, "snapshot_dir": str(tmp_path)}, - context=SimpleNamespace(workload="workload"), - ) - assert live is not None and live.on_round is not None - live.on_round(3, []) - wait_for(lambda: load(tmp_path)["progress"]["round"] == 3) - assert live.on_complete is not None - live.on_complete() - - create_adapter().write( - {**PLUGIN_CONFIG, "snapshot_dir": str(tmp_path)}, - result=_final_result(4), - output_dir=tmp_path, - ) - - snapshot = load(tmp_path) - assert snapshot["run"]["terminal"] is True - assert snapshot["progress"] == {"round": 3, "evaluated": 4} - - def test_plugin_write_without_a_live_snapshot_starts_at_round_zero( tmp_path: Path, renders: list[str], fake_search_config: None ) -> None: @@ -933,3 +908,34 @@ class LegacyCallbacks: ) assert callbacks is None assert not (tmp_path / SNAPSHOT_FILE_NAME).exists() + + +def test_a_persistent_write_failure_does_not_spin_the_writer( + tmp_path: Path, renders: list[str], monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(adapter_module, "_WRITE_RETRY_BACKOFF", 0.2) + adapter = DGDRRunOutputAdapter(make_config(tmp_path), workload=None) + adapter.start() + attempts = {"count": 0} + + def failing(directory: Path, snapshot: Any) -> Path: + attempts["count"] += 1 + raise OSError("disk full") + + monkeypatch.setattr(adapter_module, "write_snapshot", failing) + adapter.on_candidate(record("a", 1.0)) + time.sleep(0.6) + seen = attempts["count"] + monkeypatch.undo() + adapter.close() + assert 1 <= seen <= 5 + + +def test_plugin_write_keeps_the_objectives_of_the_selected_candidates( + tmp_path: Path, renders: list[str], fake_search_config: None +) -> None: + result = _final_result(1) + result.selected_candidates = [record("a", 9.0, objectives={"throughput": 12.5})] + create_adapter().write(PLUGIN_CONFIG, result=result, output_dir=tmp_path) + (candidate,) = load(tmp_path)["candidates"] + assert candidate["metrics"]["objectives"] == {"throughput": 12.5} From 5b0590bf40763dc7661dc8b8de5602c6d7e2c2b2 Mon Sep 17 00:00:00 2001 From: devivasudevan <49675305+devivasudevan@users.noreply.github.com> Date: Thu, 8 Oct 2026 16:51:19 +0200 Subject: [PATCH 10/40] test(aisimulate): drop redundant snapshot tests --- .../src/dynamo/aisimulate/tests/dgdr_run/test_snapshot.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/components/src/dynamo/aisimulate/tests/dgdr_run/test_snapshot.py b/components/src/dynamo/aisimulate/tests/dgdr_run/test_snapshot.py index 107f79a8e0a6..6ea05b6536e2 100644 --- a/components/src/dynamo/aisimulate/tests/dgdr_run/test_snapshot.py +++ b/components/src/dynamo/aisimulate/tests/dgdr_run/test_snapshot.py @@ -35,12 +35,6 @@ def materialized(candidate_id: str = "evaluated-point-a") -> SnapshotCandidate: ) -def test_only_running_is_a_non_terminal_phase() -> None: - assert not RunPhase.RUNNING.terminal - assert RunPhase.SUCCEEDED.terminal - assert RunPhase.FAILED.terminal - - def test_wire_format_of_a_running_snapshot() -> None: payload = DGDRRunSnapshot( phase=RunPhase.RUNNING, @@ -124,7 +118,7 @@ def test_inconsistent_candidates_are_rejected(kwargs: dict) -> None: def test_write_replaces_the_file_atomically_and_leaves_no_temporary_files( tmp_path: Path, ) -> None: - for round_no in range(5): + for round_no in range(2): path = write_snapshot( tmp_path, DGDRRunSnapshot(phase=RunPhase.RUNNING, round_no=round_no, evaluated=0), From f1952c7e22373ca6ed3a2b7dc79a97f84914a9b6 Mon Sep 17 00:00:00 2001 From: devivasudevan <49675305+devivasudevan@users.noreply.github.com> Date: Thu, 8 Oct 2026 16:59:52 +0200 Subject: [PATCH 11/40] fix(operator): include run UID in candidate names --- .../internal/dgdrrunpublisher/kube.go | 9 +++ .../internal/dgdrrunpublisher/publisher.go | 36 +++++++---- .../dgdrrunpublisher/publisher_test.go | 62 ++++++++++--------- 3 files changed, 66 insertions(+), 41 deletions(-) diff --git a/deploy/operator/internal/dgdrrunpublisher/kube.go b/deploy/operator/internal/dgdrrunpublisher/kube.go index 62e1a21fe04a..49887795dfb6 100644 --- a/deploy/operator/internal/dgdrrunpublisher/kube.go +++ b/deploy/operator/internal/dgdrrunpublisher/kube.go @@ -80,6 +80,15 @@ func (k *KubeCluster) getRun(ctx context.Context) (*v1beta2.DynamoGraphDeploymen return &run, nil } +// RunUID is the UID of the current incarnation of the run; candidate names include it. +func (k *KubeCluster) RunUID(ctx context.Context) (string, error) { + run, err := k.getRun(ctx) + if err != nil { + return "", err + } + return string(run.UID), nil +} + func (k *KubeCluster) ListCandidates(ctx context.Context) ([]dgdcreconcile.CurrentDGDC, error) { run, err := k.getRun(ctx) if err != nil { diff --git a/deploy/operator/internal/dgdrrunpublisher/publisher.go b/deploy/operator/internal/dgdrrunpublisher/publisher.go index 0f3c011e9468..500d05f1f6b4 100644 --- a/deploy/operator/internal/dgdrrunpublisher/publisher.go +++ b/deploy/operator/internal/dgdrrunpublisher/publisher.go @@ -114,6 +114,8 @@ type Cluster interface { CreateCandidate(ctx context.Context, name string, candidate dgdcreconcile.DesiredCandidate) error DeleteCandidate(ctx context.Context, name string) error PatchRunStatus(ctx context.Context, status RunStatus) error + // RunUID identifies the current incarnation of the run, which candidate names include. + RunUID(ctx context.Context) (string, error) // SweeperState reads the Sweeper container's status from this pod. SweeperState(ctx context.Context) (SweeperState, error) } @@ -135,24 +137,32 @@ type Publisher struct { // maxObjectName is the Kubernetes limit on an object name (DNS subdomain). const maxObjectName = 253 -// CandidateName is the DGDC name for a candidate id. It depends only on the run and the -// stable id, never on rank. A run name too long to leave room for the id is truncated -// and a hash of the full run name is added, so names stay unique and within the limit. -func CandidateName(runName, id string) string { - name := runName + "-" + id - if len(name) <= maxObjectName { - return name +// CandidateName is the DGDC name for a candidate id. It depends only on the run +// incarnation (name and UID) and the stable id, never on rank. The UID component keeps a +// recreated run of the same name from colliding with candidates of the previous one that +// are still waiting for garbage collection. A run name too long to leave room for the +// rest is truncated and a hash of the full run name is added, so names stay unique and +// within the limit. +func CandidateName(runName, runUID, id string) string { + uidSum := sha256.Sum256([]byte(runUID)) + suffix := "-" + hex.EncodeToString(uidSum[:4]) + "-" + id + if len(runName)+len(suffix) <= maxObjectName { + return runName + suffix } - sum := sha256.Sum256([]byte(runName)) - hash := hex.EncodeToString(sum[:6]) - room := maxObjectName - len(id) - len(hash) - 2 - return strings.TrimRight(runName[:room], "-.") + "-" + hash + "-" + id + nameSum := sha256.Sum256([]byte(runName)) + hash := hex.EncodeToString(nameSum[:6]) + room := maxObjectName - len(suffix) - len(hash) - 1 + return strings.TrimRight(runName[:room], "-.") + "-" + hash + suffix } // Reconcile applies one snapshot: create missing candidates, patch the run status // (ordered references and progress), then delete obsolete candidates. Existing // candidates are never updated. It is idempotent and safe to retry. func (p *Publisher) Reconcile(ctx context.Context, snap *Snapshot) error { + runUID, err := p.Cluster.RunUID(ctx) + if err != nil { + return fmt.Errorf("reading run: %w", err) + } var desired []dgdcreconcile.DesiredCandidate var names []string failed := 0 @@ -167,7 +177,7 @@ func (p *Publisher) Reconcile(ctx context.Context, snap *Snapshot) error { Parameters: candidate.Parameters, Metrics: candidate.Metrics, }) - names = append(names, CandidateName(p.RunName, candidate.ID)) + names = append(names, CandidateName(p.RunName, runUID, candidate.ID)) } current, err := p.Cluster.ListCandidates(ctx) @@ -179,7 +189,7 @@ func (p *Publisher) Reconcile(ctx context.Context, snap *Snapshot) error { return err } for _, candidate := range actions.Creates { - if err := p.Cluster.CreateCandidate(ctx, CandidateName(p.RunName, candidate.ID), candidate); err != nil { + if err := p.Cluster.CreateCandidate(ctx, CandidateName(p.RunName, runUID, candidate.ID), candidate); err != nil { return fmt.Errorf("creating candidate %s: %w", candidate.ID, err) } } diff --git a/deploy/operator/internal/dgdrrunpublisher/publisher_test.go b/deploy/operator/internal/dgdrrunpublisher/publisher_test.go index 35814c6f8ed2..dfd3608f9334 100644 --- a/deploy/operator/internal/dgdrrunpublisher/publisher_test.go +++ b/deploy/operator/internal/dgdrrunpublisher/publisher_test.go @@ -97,6 +97,8 @@ func (f *fakeCluster) PatchRunStatus(_ context.Context, s RunStatus) error { return nil } +func (f *fakeCluster) RunUID(context.Context) (string, error) { return testRunUID, nil } + func (f *fakeCluster) SweeperState(context.Context) (SweeperState, error) { f.mu.Lock() defer f.mu.Unlock() @@ -111,6 +113,10 @@ func (f *fakeCluster) setState(s SweeperState) { const validManifest = "apiVersion: nvidia.com/v1beta1\nkind: DynamoGraphDeployment\nspec: {}\n" +const testRunUID = "uid-1" + +func cn(id string) string { return CandidateName("run", testRunUID, id) } + type cand struct { id string failed bool @@ -180,13 +186,13 @@ func TestSwapLeavesBothCandidatesUntouched(t *testing.T) { if err := p.Reconcile(ctx, mustParse(t, snapshotJSON(t, PhaseRunning, cand{id: "b", metrics: map[string]any{"score": 3.0}}, cand{id: "a", metrics: map[string]any{"score": 2.0}}))); err != nil { t.Fatal(err) } - if !reflect.DeepEqual(fc.creates, []string{"run-a", "run-b"}) || len(fc.deletes) != 0 { + if !reflect.DeepEqual(fc.creates, []string{cn("a"), cn("b")}) || len(fc.deletes) != 0 { t.Fatalf("swap touched DGDCs: creates=%v deletes=%v", fc.creates, fc.deletes) } if !reflect.DeepEqual(before, fc.created) { t.Fatalf("a published DGDC's content changed: before=%v after=%v", before, fc.created) } - if got, want := fc.patches[len(fc.patches)-1].CandidateNames, []string{"run-b", "run-a"}; !reflect.DeepEqual(got, want) { + if got, want := fc.patches[len(fc.patches)-1].CandidateNames, []string{cn("b"), cn("a")}; !reflect.DeepEqual(got, want) { t.Fatalf("ordered references = %v, want %v", got, want) } } @@ -196,7 +202,7 @@ func TestEvaluatedPointReachesTheClusterComplete(t *testing.T) { if err := p.Reconcile(context.Background(), mustParse(t, snapshotJSON(t, PhaseRunning, cand{id: "a", metrics: map[string]any{"goodputPerGpu": 87.7}}))); err != nil { t.Fatal(err) } - got := fc.created["run-a"] + got := fc.created[cn("a")] if got.Spec != validManifest { t.Fatalf("manifest = %q", got.Spec) } @@ -222,14 +228,14 @@ func TestReturningCandidateIsRecreatedIdentically(t *testing.T) { t.Fatal(err) } if i == 0 { - firstB = fc.created["run-b"] + firstB = fc.created[cn("b")] } } - if !reflect.DeepEqual(fc.creates, []string{"run-a", "run-b", "run-b"}) || !reflect.DeepEqual(fc.deletes, []string{"run-b"}) { + if !reflect.DeepEqual(fc.creates, []string{cn("a"), cn("b"), cn("b")}) || !reflect.DeepEqual(fc.deletes, []string{cn("b")}) { t.Fatalf("creates=%v deletes=%v", fc.creates, fc.deletes) } - if !reflect.DeepEqual(firstB, fc.created["run-b"]) { - t.Fatalf("recreated candidate differs: %v vs %v", firstB, fc.created["run-b"]) + if !reflect.DeepEqual(firstB, fc.created[cn("b")]) { + t.Fatalf("recreated candidate differs: %v vs %v", firstB, fc.created[cn("b")]) } } @@ -252,13 +258,13 @@ func TestDroppedCandidateIsDeletedAfterStatusPatch(t *testing.T) { ctx := context.Background() _ = p.Reconcile(ctx, mustParse(t, snapshotJSON(t, PhaseRunning, cand{id: "a"}, cand{id: "b"}))) _ = p.Reconcile(ctx, mustParse(t, snapshotJSON(t, PhaseRunning, cand{id: "a"}))) - if !reflect.DeepEqual(fc.deletes, []string{"run-b"}) { + if !reflect.DeepEqual(fc.deletes, []string{cn("b")}) { t.Fatalf("deletes = %v", fc.deletes) } - if got := fc.patches[len(fc.patches)-1].CandidateNames; !reflect.DeepEqual(got, []string{"run-a"}) { + if got := fc.patches[len(fc.patches)-1].CandidateNames; !reflect.DeepEqual(got, []string{cn("a")}) { t.Fatalf("status refs = %v", got) } - want := []string{"create:run-a", "create:run-b", "patch", "patch", "delete:run-b"} + want := []string{"create:" + cn("a"), "create:" + cn("b"), "patch", "patch", "delete:" + cn("b")} if !reflect.DeepEqual(fc.ops, want) { t.Fatalf("ops = %v, want %v", fc.ops, want) } @@ -269,10 +275,10 @@ func TestFailedMaterializationIsNotCreatedButCounted(t *testing.T) { if err := p.Reconcile(context.Background(), mustParse(t, snapshotJSON(t, PhaseRunning, cand{id: "a"}, cand{id: "x", failed: true}))); err != nil { t.Fatal(err) } - if !reflect.DeepEqual(fc.creates, []string{"run-a"}) { + if !reflect.DeepEqual(fc.creates, []string{cn("a")}) { t.Fatalf("creates = %v", fc.creates) } - if got := fc.patches[0].CandidateNames; !reflect.DeepEqual(got, []string{"run-a"}) { + if got := fc.patches[0].CandidateNames; !reflect.DeepEqual(got, []string{cn("a")}) { t.Fatalf("refs must not include the failed candidate: %v", got) } if msg := fc.patches[0].Message; !strings.Contains(msg, "1 candidate(s) failed materialization") { @@ -395,7 +401,7 @@ func TestRunTerminalSnapshotAcknowledges(t *testing.T) { if err != nil || ExitCode(err) != ExitAcknowledged { t.Fatalf("err = %v", err) } - if got := lastPatch(t, fc).CandidateNames; !reflect.DeepEqual(got, []string{"run-a"}) { + if got := lastPatch(t, fc).CandidateNames; !reflect.DeepEqual(got, []string{cn("a")}) { t.Fatalf("refs = %v", got) } } @@ -407,7 +413,7 @@ func TestRunTerminalFailedSnapshotIsReconciledThenFails(t *testing.T) { if !errors.Is(err, ErrRunFailed) || ExitCode(err) != ExitRunFailed { t.Fatalf("err = %v", err) } - if !reflect.DeepEqual(fc.creates, []string{"run-a"}) { + if !reflect.DeepEqual(fc.creates, []string{cn("a")}) { t.Fatalf("creates = %v", fc.creates) } } @@ -436,7 +442,7 @@ func TestRunCrashWithoutTerminalSnapshotReconcilesLastSnapshot(t *testing.T) { if err := runWithTimeout(t, p); err != nil { t.Fatal(err) } - if got := lastPatch(t, fc).CandidateNames; !reflect.DeepEqual(got, []string{"run-a"}) { + if got := lastPatch(t, fc).CandidateNames; !reflect.DeepEqual(got, []string{cn("a")}) { t.Fatalf("refs = %v", got) } } @@ -463,7 +469,7 @@ func TestRunCleanExitWithoutTerminalSnapshotIsProtocolViolation(t *testing.T) { if !errors.Is(err, ErrMissingTerminalSnapshot) || ExitCode(err) != ExitProtocolViolation { t.Fatalf("err = %v", err) } - if !reflect.DeepEqual(fc.creates, []string{"run-a"}) { + if !reflect.DeepEqual(fc.creates, []string{cn("a")}) { t.Fatalf("last snapshot must still be reconciled: %v", fc.creates) } } @@ -503,7 +509,7 @@ func TestRunTerminalSnapshotReconcileFailureIsRetriedUntilSuccess(t *testing.T) if err := <-done; err != nil { t.Fatal(err) } - if !reflect.DeepEqual(fc.creates, []string{"run-a"}) { + if !reflect.DeepEqual(fc.creates, []string{cn("a")}) { t.Fatalf("creates = %v", fc.creates) } } @@ -529,15 +535,15 @@ func TestRunContextCancelledReturnsError(t *testing.T) { } } -func TestCandidateNameIsBoundedAndUnique(t *testing.T) { - if got := CandidateName("run", "a"); got != "run-a" { - t.Fatalf("short names are unchanged: %q", got) - } +func TestCandidateNameIsBoundedAndUniquePerRunIncarnation(t *testing.T) { id := "evaluated-point-0123456789ab" + if cn("a") != CandidateName("run", testRunUID, "a") || cn("a") == CandidateName("run", "uid-2", "a") { + t.Fatalf("a recreated run must get different names: %q", cn("a")) + } long := strings.Repeat("r", 253) other := strings.Repeat("r", 252) + "s" - name := CandidateName(long, id) - if len(name) > 253 || !strings.HasSuffix(name, "-"+id) || name == CandidateName(other, id) || name != CandidateName(long, id) { + name := CandidateName(long, testRunUID, id) + if len(name) > 253 || !strings.HasSuffix(name, "-"+id) || name == CandidateName(other, testRunUID, id) || name != CandidateName(long, testRunUID, id) { t.Fatalf("name = %q (%d)", name, len(name)) } } @@ -549,11 +555,11 @@ func TestUnchangedSnapshotRepairsDrift(t *testing.T) { if err := p.syncOnce(ctx); err != nil { t.Fatal(err) } - delete(fc.candidates, "run-a") // someone deletes the referenced candidate + delete(fc.candidates, cn("a")) // someone deletes the referenced candidate if err := p.syncOnce(ctx); err != nil { t.Fatal(err) } - if !reflect.DeepEqual(fc.creates, []string{"run-a", "run-a"}) { + if !reflect.DeepEqual(fc.creates, []string{cn("a"), cn("a")}) { t.Fatalf("creates = %v", fc.creates) } } @@ -561,13 +567,13 @@ func TestUnchangedSnapshotRepairsDrift(t *testing.T) { func TestCandidateWithoutStatusIsCompletedBeforeItIsReferenced(t *testing.T) { p, fc, dir := newPublisher(t) ctx := context.Background() - fc.candidates["run-a"] = "a" - fc.incomplete["run-a"] = true // created, but the status write failed + fc.candidates[cn("a")] = "a" + fc.incomplete[cn("a")] = true // created, but the status write failed writeSnapshot(t, dir, snapshotJSON(t, PhaseRunning, cand{id: "a"})) if err := p.syncOnce(ctx); err != nil { t.Fatal(err) } - if want := []string{"create:run-a", "patch"}; !reflect.DeepEqual(fc.ops, want) { + if want := []string{"create:" + cn("a"), "patch"}; !reflect.DeepEqual(fc.ops, want) { t.Fatalf("ops = %v, want %v", fc.ops, want) } } From 58cd3c4f41775f48ab3639b56a2cef0a909cb42e Mon Sep 17 00:00:00 2001 From: devivasudevan <49675305+devivasudevan@users.noreply.github.com> Date: Thu, 8 Oct 2026 22:21:55 +0200 Subject: [PATCH 12/40] fix(aisimulate): pass used_gpus to the DGD renderer; trim tests --- .../aisimulate/output/dgdr_run/adapter.py | 5 +-- .../aisimulate/tests/dgdr_run/test_adapter.py | 31 +++++++++++++------ 2 files changed, 24 insertions(+), 12 deletions(-) diff --git a/components/src/dynamo/aisimulate/output/dgdr_run/adapter.py b/components/src/dynamo/aisimulate/output/dgdr_run/adapter.py index 2f493a5c075f..112c1331ae40 100644 --- a/components/src/dynamo/aisimulate/output/dgdr_run/adapter.py +++ b/components/src/dynamo/aisimulate/output/dgdr_run/adapter.py @@ -502,7 +502,7 @@ def _materialize(self, item: _Retained) -> SnapshotCandidate: self._render_attempts[item.id] = self._render_attempts.get(item.id, 0) + 1 try: manifest = render_dgd( - _RenderInput(config=item.parameters), + _RenderInput(config=item.parameters, used_gpus=item.used_gpus), self._workload, self._options, dgd_name=f"{self._config.name}-{item.id.removeprefix(_ID_PREFIX)}", @@ -531,9 +531,10 @@ def _materialize(self, item: _Retained) -> SnapshotCandidate: @dataclass(frozen=True) class _RenderInput: - """What ``render_dgd`` reads from a candidate: its resolved config.""" + """The evaluated candidate as the renderer sees it: its config and GPU count.""" config: dict[str, Any] + used_gpus: int class DGDRRunOutputPlugin: diff --git a/components/src/dynamo/aisimulate/tests/dgdr_run/test_adapter.py b/components/src/dynamo/aisimulate/tests/dgdr_run/test_adapter.py index 4060646bea09..75cc68cb5d38 100644 --- a/components/src/dynamo/aisimulate/tests/dgdr_run/test_adapter.py +++ b/components/src/dynamo/aisimulate/tests/dgdr_run/test_adapter.py @@ -235,7 +235,7 @@ def test_a_new_better_point_reorders_but_does_not_rematerialize_retained_ones( after = {c["id"]: c["manifest"] for c in load(tmp_path)["candidates"]} assert {key: after[key] for key in before} == before - assert sorted(renders) == ["a", "b", "c"] # only the new point was rendered + assert sorted(renders) == ["a", "b", "c"] def test_progress_updates_do_not_rematerialize( @@ -342,7 +342,7 @@ def test_the_terminal_snapshot_contains_every_update_accepted_before_close( with DGDRRunOutputAdapter(config, workload=None) as adapter: for index in range(50): adapter.on_candidate(record(f"w{index}", score=1.0 + index)) - adapter.on_candidate(record("winner", score=1000.0)) # right before close + adapter.on_candidate(record("winner", score=1000.0)) snapshot = load(tmp_path) assert snapshot["run"]["terminal"] is True @@ -391,14 +391,6 @@ def test_updates_after_close_are_ignored_and_close_is_idempotent( assert adapter.snapshot_writes == writes -def test_close_works_when_start_was_never_called( - tmp_path: Path, renders: list[str] -) -> None: - adapter = DGDRRunOutputAdapter(make_config(tmp_path), workload=None) - adapter.close(phase=RunPhase.FAILED, error="setup failed") - assert load(tmp_path)["run"]["phase"] == "Failed" - - def test_close_rejects_a_non_terminal_phase(tmp_path: Path) -> None: adapter = DGDRRunOutputAdapter(make_config(tmp_path), workload=None) with pytest.raises(ValueError, match="terminal"): @@ -939,3 +931,22 @@ def test_plugin_write_keeps_the_objectives_of_the_selected_candidates( create_adapter().write(PLUGIN_CONFIG, result=result, output_dir=tmp_path) (candidate,) = load(tmp_path)["candidates"] assert candidate["metrics"]["objectives"] == {"throughput": 12.5} + + +def test_the_renderer_receives_the_candidates_gpu_count( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + seen: list[Any] = [] + + def capture(candidate: Any, *_: Any, **kwargs: Any) -> str: + seen.append(candidate) + return "kind: DynamoGraphDeployment\n" + + monkeypatch.setattr(adapter_module, "render_dgd", capture) + adapter = DGDRRunOutputAdapter(make_config(tmp_path), workload=None) + adapter.on_candidate(record("a", 9.0, gpus=4)) + adapter.close() + + (candidate,) = seen + assert candidate.used_gpus == 4 + assert candidate.config["tag"] == "a" From 3d98cd3d4f3d5b68e158aba46867bc9e6d51aca4 Mon Sep 17 00:00:00 2001 From: devivasudevan <49675305+devivasudevan@users.noreply.github.com> Date: Mon, 12 Oct 2026 13:14:36 +0200 Subject: [PATCH 13/40] refactor(operator): run the DGDR publisher as a controller-runtime controller Split the publisher into an input-observation loop and a Reconciler that share one workqueue key (the Run's namespace/name). The observer stores semantic snapshot changes atomically and backs off while input is unchanged. The Reconciler creates missing candidates with server-side apply, populates candidate status, updates run progress and ordered references only where they differ, then deletes obsolete candidates. The manager cache is scoped to this run, its candidates and its own pod. Sidecar lifecycle, SweeperState check, exit semantics and Completed ownership are unchanged. Signed-off-by: devivasudevan <49675305+devivasudevan@users.noreply.github.com> --- .../operator/cmd/dgdr-run-publisher/main.go | 58 +- .../operator/internal/dgdcreconcile/diff.go | 16 +- .../internal/dgdcreconcile/diff_test.go | 32 +- .../operator/internal/dgdrrunpublisher/doc.go | 56 ++ .../internal/dgdrrunpublisher/input.go | 190 +++++ .../internal/dgdrrunpublisher/input_test.go | 179 +++++ .../internal/dgdrrunpublisher/kube.go | 280 -------- .../internal/dgdrrunpublisher/manager.go | 208 ++++++ .../internal/dgdrrunpublisher/outcome.go | 114 +++ .../internal/dgdrrunpublisher/outcome_test.go | 78 ++ .../internal/dgdrrunpublisher/plan.go | 161 +++++ .../internal/dgdrrunpublisher/plan_test.go | 123 ++++ .../internal/dgdrrunpublisher/publisher.go | 378 ---------- .../dgdrrunpublisher/publisher_test.go | 666 ------------------ .../internal/dgdrrunpublisher/reconciler.go | 402 +++++++++++ .../dgdrrunpublisher/reconciler_test.go | 101 +++ .../dgdrrunpublisher/snapshot_test.go | 158 +++++ 17 files changed, 1814 insertions(+), 1386 deletions(-) create mode 100644 deploy/operator/internal/dgdrrunpublisher/doc.go create mode 100644 deploy/operator/internal/dgdrrunpublisher/input.go create mode 100644 deploy/operator/internal/dgdrrunpublisher/input_test.go delete mode 100644 deploy/operator/internal/dgdrrunpublisher/kube.go create mode 100644 deploy/operator/internal/dgdrrunpublisher/manager.go create mode 100644 deploy/operator/internal/dgdrrunpublisher/outcome.go create mode 100644 deploy/operator/internal/dgdrrunpublisher/outcome_test.go create mode 100644 deploy/operator/internal/dgdrrunpublisher/plan.go create mode 100644 deploy/operator/internal/dgdrrunpublisher/plan_test.go delete mode 100644 deploy/operator/internal/dgdrrunpublisher/publisher.go delete mode 100644 deploy/operator/internal/dgdrrunpublisher/publisher_test.go create mode 100644 deploy/operator/internal/dgdrrunpublisher/reconciler.go create mode 100644 deploy/operator/internal/dgdrrunpublisher/reconciler_test.go create mode 100644 deploy/operator/internal/dgdrrunpublisher/snapshot_test.go diff --git a/deploy/operator/cmd/dgdr-run-publisher/main.go b/deploy/operator/cmd/dgdr-run-publisher/main.go index 8140905a4dd0..16f446888347 100644 --- a/deploy/operator/cmd/dgdr-run-publisher/main.go +++ b/deploy/operator/cmd/dgdr-run-publisher/main.go @@ -15,7 +15,8 @@ * limitations under the License. */ -// Command dgdr-run-publisher is the sidecar of a DGDR(v2) run Job. It watches the +// Command dgdr-run-publisher is the sidecar of a DGDR(v2) run Job. It is a +// controller-runtime controller for one DynamoGraphDeploymentRun: it observes the // Sweeper's pod-local snapshot file and reconciles it into DynamoGraphDeploymentCandidates // and DynamoGraphDeploymentRun.status. See internal/dgdrrunpublisher. package main @@ -30,13 +31,8 @@ import ( "syscall" "time" - v1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" "github.com/ai-dynamo/dynamo/deploy/operator/internal/dgdrrunpublisher" - v1beta2 "github.com/ai-dynamo/dynamo/deploy/operator/internal/dgdrrunpublisher/placeholderapi" - "k8s.io/apimachinery/pkg/runtime" - clientgoscheme "k8s.io/client-go/kubernetes/scheme" ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/client" ) func main() { @@ -44,53 +40,25 @@ func main() { } func run() int { - var ( - snapshotDir string - runName string - sweeperContainer string - pollInterval time.Duration - ) - flag.StringVar(&snapshotDir, "snapshot-dir", "", "directory shared with the Sweeper container") - flag.StringVar(&runName, "run-name", "", "name of the DynamoGraphDeploymentRun") - flag.StringVar(&sweeperContainer, "sweeper-container", "sweeper", "name of the Sweeper container in this pod") - flag.DurationVar(&pollInterval, "poll-interval", time.Second, "snapshot poll interval") + var cfg dgdrrunpublisher.Config + flag.StringVar(&cfg.SnapshotDir, "snapshot-dir", "", "directory shared with the Sweeper container") + flag.StringVar(&cfg.RunName, "run-name", "", "name of the DynamoGraphDeploymentRun") + flag.StringVar(&cfg.RunUID, "run-uid", "", "UID of the DynamoGraphDeploymentRun this pod serves") + flag.StringVar(&cfg.SweeperContainer, "sweeper-container", "sweeper", "name of the Sweeper container in this pod") + flag.DurationVar(&cfg.MinPollInterval, "poll-interval", time.Second, "snapshot check interval after a change") + flag.DurationVar(&cfg.MaxPollInterval, "max-poll-interval", 5*time.Second, "longest snapshot check interval while the snapshot is unchanged") flag.Parse() - namespace, podName := os.Getenv("POD_NAMESPACE"), os.Getenv("POD_NAME") - if snapshotDir == "" || runName == "" || namespace == "" || podName == "" { - fmt.Fprintln(os.Stderr, "--snapshot-dir, --run-name and the POD_NAMESPACE/POD_NAME environment variables are required") - return dgdrrunpublisher.ExitReconcileFailed - } - - scheme := runtime.NewScheme() - for _, add := range []func(*runtime.Scheme) error{clientgoscheme.AddToScheme, v1beta1.AddToScheme, v1beta2.AddToScheme} { - if err := add(scheme); err != nil { - log.Printf("building scheme: %v", err) - return dgdrrunpublisher.ExitReconcileFailed - } - } - kube, err := client.New(ctrl.GetConfigOrDie(), client.Options{Scheme: scheme}) - if err != nil { - log.Printf("creating client: %v", err) + cfg.Namespace, cfg.PodName = os.Getenv("POD_NAMESPACE"), os.Getenv("POD_NAME") + if cfg.SnapshotDir == "" || cfg.RunName == "" || cfg.RunUID == "" || cfg.Namespace == "" || cfg.PodName == "" { + fmt.Fprintln(os.Stderr, "--snapshot-dir, --run-name, --run-uid and the POD_NAMESPACE/POD_NAME environment variables are required") return dgdrrunpublisher.ExitReconcileFailed } ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGTERM, syscall.SIGINT) defer stop() - publisher := &dgdrrunpublisher.Publisher{ - Cluster: &dgdrrunpublisher.KubeCluster{ - Client: kube, - Namespace: namespace, - RunName: runName, - PodName: podName, - SweeperContainer: sweeperContainer, - }, - SnapshotDir: snapshotDir, - RunName: runName, - PollInterval: pollInterval, - } - err = publisher.Run(ctx) + err := dgdrrunpublisher.Run(ctx, ctrl.GetConfigOrDie(), cfg) if err != nil { log.Printf("publisher: %v", err) } diff --git a/deploy/operator/internal/dgdcreconcile/diff.go b/deploy/operator/internal/dgdcreconcile/diff.go index fcf36906c89d..b9de85216190 100644 --- a/deploy/operator/internal/dgdcreconcile/diff.go +++ b/deploy/operator/internal/dgdcreconcile/diff.go @@ -61,16 +61,20 @@ type CurrentDGDC struct { Name string ID string // Incomplete means the DGDC exists but its initial status was never populated (for - // example the process stopped between creating it and writing its status). + // example the process stopped between creating it and writing its status). Its spec + // is immutable, so only the status is completed. Incomplete bool } // Actions is what the caller must apply. There are no updates: DGDCs are immutable. type Actions struct { - // Creates are idempotent "ensure" calls: they also cover a desired candidate whose - // DGDC exists but is Incomplete, so its status is finished before it is referenced. + // Creates are desired candidates with no DGDC yet. Creates []DesiredCandidate - Deletes []string // DGDC names + // Incomplete are desired candidates whose DGDC exists but whose initial status was + // never populated. The caller finishes the status (never the spec) before the + // candidate is referenced from the run. + Incomplete []DesiredCandidate + Deletes []string // DGDC names } // ComputeActions diffs desired against current. Output order follows the input order @@ -101,8 +105,10 @@ func ComputeActions(desired []DesiredCandidate, current []CurrentDGDC) (Actions, var actions Actions for _, candidate := range desired { - if _, exists := currentIDs[candidate.ID]; !exists || incomplete[candidate.ID] { + if _, exists := currentIDs[candidate.ID]; !exists { actions.Creates = append(actions.Creates, candidate) + } else if incomplete[candidate.ID] { + actions.Incomplete = append(actions.Incomplete, candidate) } } for _, existing := range current { diff --git a/deploy/operator/internal/dgdcreconcile/diff_test.go b/deploy/operator/internal/dgdcreconcile/diff_test.go index 3d5eda01441c..3d0f155e2173 100644 --- a/deploy/operator/internal/dgdcreconcile/diff_test.go +++ b/deploy/operator/internal/dgdcreconcile/diff_test.go @@ -49,19 +49,20 @@ func createIDs(a Actions) []string { func TestComputeActions(t *testing.T) { tests := []struct { - name string - desired []DesiredCandidate - current []CurrentDGDC - wantCreates []string - wantDeletes []string + name string + desired []DesiredCandidate + current []CurrentDGDC + wantCreates []string + wantIncomplete []string + wantDeletes []string }{ - {"empty both", nil, nil, nil, nil}, - {"all new", d("a", "b"), nil, []string{"a", "b"}, nil}, - {"all gone", nil, c("n-a", "a", "n-b", "b"), nil, []string{"n-a", "n-b"}}, - {"steady state needs no action", d("a", "b"), c("n-a", "a", "n-b", "b"), nil, nil}, - {"reordering desired needs no action", d("b", "a"), c("n-a", "a", "n-b", "b"), nil, nil}, - {"incomplete DGDC is ensured again", d("a", "b"), []CurrentDGDC{{Name: "n-a", ID: "a"}, {Name: "n-b", ID: "b", Incomplete: true}}, []string{"b"}, nil}, - {"mixed", d("a", "c"), c("n-a", "a", "n-b", "b"), []string{"c"}, []string{"n-b"}}, + {"empty both", nil, nil, nil, nil, nil}, + {"all new", d("a", "b"), nil, []string{"a", "b"}, nil, nil}, + {"all gone", nil, c("n-a", "a", "n-b", "b"), nil, nil, []string{"n-a", "n-b"}}, + {"steady state needs no action", d("a", "b"), c("n-a", "a", "n-b", "b"), nil, nil, nil}, + {"reordering desired needs no action", d("b", "a"), c("n-a", "a", "n-b", "b"), nil, nil, nil}, + {"incomplete DGDC gets its status finished, not a new spec", d("a", "b"), []CurrentDGDC{{Name: "n-a", ID: "a"}, {Name: "n-b", ID: "b", Incomplete: true}}, nil, []string{"b"}, nil}, + {"mixed", d("a", "c"), c("n-a", "a", "n-b", "b"), []string{"c"}, nil, []string{"n-b"}}, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { @@ -72,6 +73,13 @@ func TestComputeActions(t *testing.T) { if !reflect.DeepEqual(createIDs(got), tc.wantCreates) { t.Errorf("creates = %v, want %v", createIDs(got), tc.wantCreates) } + var gotIncomplete []string + for _, x := range got.Incomplete { + gotIncomplete = append(gotIncomplete, x.ID) + } + if !reflect.DeepEqual(gotIncomplete, tc.wantIncomplete) { + t.Errorf("incomplete = %v, want %v", gotIncomplete, tc.wantIncomplete) + } if !reflect.DeepEqual(got.Deletes, tc.wantDeletes) { t.Errorf("deletes = %v, want %v", got.Deletes, tc.wantDeletes) } diff --git a/deploy/operator/internal/dgdrrunpublisher/doc.go b/deploy/operator/internal/dgdrrunpublisher/doc.go new file mode 100644 index 000000000000..ac6cc837ab8c --- /dev/null +++ b/deploy/operator/internal/dgdrrunpublisher/doc.go @@ -0,0 +1,56 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package dgdrrunpublisher is the sidecar that turns the Sweeper's pod-local desired +// state snapshot into Kubernetes state: it creates/deletes DynamoGraphDeploymentCandidates +// and updates the ordered candidate references and progress on +// DynamoGraphDeploymentRun.status. +// +// It is a controller-runtime controller for exactly one DynamoGraphDeploymentRun, with +// two loops that meet at one workqueue key (the Run's namespace/name): +// +// 1. The input observer (Observer) checks the snapshot file the Sweeper writes into a +// shared emptyDir. On a semantic change it atomically stores the new desired state +// (InputStore) and enqueues the key. Unchanged input enqueues nothing and backs the +// check interval off; a change resets it. +// 2. The Reconciler consumes the key. Informer events for the Run, its candidates and +// this Pod enqueue the same key. It reads the latest stored input and the cached +// cluster state, then creates missing candidates with server-side apply, completes +// candidate status, updates the Run's progress, message and ordered references only +// where they differ, and finally deletes obsolete candidates. Failed writes are +// retried by the workqueue with backoff. +// +// The manager's cache covers only this Run, its candidates (by the run-UID label) and the +// publisher's own Pod, so there is no API-server polling and no spec comparison: +// candidate specs are immutable once created. +// +// The sidecar is a regular container of the same Job pod as the Sweeper. The Job only +// completes once the publisher has exited, so a successful exit acknowledges that the +// final snapshot was reconciled. The run controller derives the terminal Completed +// condition from the Job result; the publisher never writes a phase. +// +// Termination (see Reconciler): +// +// terminal Succeeded snapshot reconciled -> exit 0 +// terminal Failed snapshot reconciled -> exit ExitRunFailed +// Sweeper exited non-zero, no terminal snapshot -> reconcile last snapshot, exit 0 +// (the Sweeper's exit fails the Job) +// Sweeper exited zero, no terminal snapshot -> reconcile last snapshot, +// exit ExitProtocolViolation +// invalid or regressing snapshot, Sweeper exited -> exit ExitProtocolViolation +// final/last snapshot cannot be reconciled -> exit ExitReconcileFailed +package dgdrrunpublisher diff --git a/deploy/operator/internal/dgdrrunpublisher/input.go b/deploy/operator/internal/dgdrrunpublisher/input.go new file mode 100644 index 000000000000..eac2093a2725 --- /dev/null +++ b/deploy/operator/internal/dgdrrunpublisher/input.go @@ -0,0 +1,190 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dgdrrunpublisher + +import ( + "context" + "errors" + "log" + "os" + "reflect" + "sync" + "time" +) + +// Input is the latest accepted desired state plus the most recent problem reading the +// input. Snapshot is never mutated after it is stored, so readers need no copy. +type Input struct { + // Snapshot is the last valid snapshot, or nil if none has been accepted yet. + Snapshot *Snapshot + // Err is why the most recent read failed (invalid or regressing snapshot, or an I/O + // error). It is cleared by the next valid read. + Err error +} + +// InputStore atomically holds the latest desired state for the reconciler. +type InputStore struct { + mu sync.RWMutex + input Input +} + +// Load returns the current input. +func (s *InputStore) Load() Input { + s.mu.RLock() + defer s.mu.RUnlock() + return s.input +} + +// accept stores snap as the latest desired state if it differs semantically from the +// current one, and reports whether it did. A snapshot that moves progress backwards is +// rejected and recorded as the input error; the last valid snapshot stays in place. +func (s *InputStore) accept(snap *Snapshot) (changed bool) { + s.mu.Lock() + defer s.mu.Unlock() + if err := checkProgress(s.input.Snapshot, snap); err != nil { + s.input.Err = err + return false + } + s.input.Err = nil + if s.input.Snapshot != nil && sameContent(s.input.Snapshot, snap) { + return false + } + s.input.Snapshot = snap + return true +} + +// fail records why the input could not be read, keeping the last valid snapshot. +func (s *InputStore) fail(err error) { + s.mu.Lock() + defer s.mu.Unlock() + s.input.Err = err +} + +// sameContent compares two snapshots ignoring the write timestamp: rewriting identical +// content with a new timestamp is not a change the cluster needs to hear about. +func sameContent(a, b *Snapshot) bool { + left, right := *a, *b + left.Timestamp, right.Timestamp = "", "" + return reflect.DeepEqual(left, right) +} + +// checkProgress rejects a snapshot that moves progress backwards relative to the +// previously accepted one: an older round, or fewer evaluated points within the same +// round. A terminal snapshot is exempt from the evaluated check because its count is the +// search's final total, not the live counter. +func checkProgress(previous, next *Snapshot) error { + if previous == nil { + return nil + } + if next.Progress.Round < previous.Progress.Round { + return violation("snapshot round regressed from %d to %d", previous.Progress.Round, next.Progress.Round) + } + if !next.Run.Terminal && next.Progress.Round == previous.Progress.Round && next.Progress.Evaluated < previous.Progress.Evaluated { + return violation("snapshot evaluated count regressed from %d to %d in round %d", previous.Progress.Evaluated, next.Progress.Evaluated, previous.Progress.Round) + } + return nil +} + +// Observer is the input-observation loop: it checks the Sweeper's snapshot file and, on a +// semantic change, stores it and enqueues the run key. It never talks to the API server. +type Observer struct { + Store *InputStore + Path string + // Enqueue asks the controller to reconcile the run. It must not block. + Enqueue func() + // MinInterval is the check interval after a change; it doubles while the input is + // unchanged, up to MaxInterval. + MinInterval time.Duration + MaxInterval time.Duration + + mu sync.Mutex // serializes Check between the loop and the reconciler + lastErr string +} + +// Check reads the snapshot once and reports whether it was a semantic change (in which +// case it has been stored and the run key enqueued). A missing file is not an error. The +// reconciler also calls it directly once the Sweeper has exited, to be certain the +// Sweeper's final write has been seen. +func (o *Observer) Check() bool { + o.mu.Lock() + defer o.mu.Unlock() + + data, err := os.ReadFile(o.Path) + if errors.Is(err, os.ErrNotExist) { + return false + } + if err != nil { + o.noteError(err) + o.Store.fail(err) + return false + } + snap, err := ParseSnapshot(data) + if err != nil { + o.noteError(err) + o.Store.fail(err) + return false + } + changed := o.Store.accept(snap) + if err := o.Store.Load().Err; err != nil { + o.noteError(err) + return false + } + o.lastErr = "" + if changed && o.Enqueue != nil { + o.Enqueue() + } + return changed +} + +// noteError logs a problem once per distinct message rather than on every check. +func (o *Observer) noteError(err error) { + if err.Error() == o.lastErr { + return + } + o.lastErr = err.Error() + log.Printf("reading snapshot: %v", err) +} + +// NextInterval is the delay before the next check: it resets to min after a change and +// otherwise doubles up to max. +func NextInterval(current, minimum, maximum time.Duration, changed bool) time.Duration { + if changed || current < minimum { + return minimum + } + next := current * 2 + if next > maximum { + return maximum + } + return next +} + +// Run checks the snapshot until ctx is done. +func (o *Observer) Run(ctx context.Context) error { + interval := o.MinInterval + timer := time.NewTimer(0) + defer timer.Stop() + for { + select { + case <-ctx.Done(): + return nil + case <-timer.C: + } + interval = NextInterval(interval, o.MinInterval, o.MaxInterval, o.Check()) + timer.Reset(interval) + } +} diff --git a/deploy/operator/internal/dgdrrunpublisher/input_test.go b/deploy/operator/internal/dgdrrunpublisher/input_test.go new file mode 100644 index 000000000000..8b13734c92fa --- /dev/null +++ b/deploy/operator/internal/dgdrrunpublisher/input_test.go @@ -0,0 +1,179 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dgdrrunpublisher + +import ( + "context" + "errors" + "os" + "path/filepath" + "testing" + "time" +) + +func newObserver(t *testing.T) (*Observer, *InputStore, string, *int) { + t.Helper() + dir := t.TempDir() + enqueued := 0 + store := &InputStore{} + return &Observer{ + Store: store, + Path: filepath.Join(dir, SnapshotFileName), + Enqueue: func() { enqueued++ }, + MinInterval: time.Millisecond, + MaxInterval: 4 * time.Millisecond, + }, store, dir, &enqueued +} + +func TestMissingSnapshotIsNotAChange(t *testing.T) { + o, store, _, enqueued := newObserver(t) + if o.Check() || *enqueued != 0 || store.Load().Snapshot != nil || store.Load().Err != nil { + t.Fatalf("missing file: input=%+v enqueued=%d", store.Load(), *enqueued) + } +} + +func TestChangedSnapshotIsStoredAndEnqueuedOnce(t *testing.T) { + o, store, dir, enqueued := newObserver(t) + writeSnapshot(t, dir, snapshotJSON(t, PhaseRunning, cand{id: "a"})) + if !o.Check() || *enqueued != 1 { + t.Fatalf("first snapshot: enqueued=%d", *enqueued) + } + if store.Load().Snapshot == nil || store.Load().Snapshot.Candidates[0].ID != "a" { + t.Fatalf("snapshot not stored: %+v", store.Load()) + } + if o.Check() || *enqueued != 1 { + t.Fatalf("unchanged input must not enqueue again: enqueued=%d", *enqueued) + } + writeSnapshot(t, dir, snapshotJSON(t, PhaseRunning, cand{id: "a"}, cand{id: "b"})) + if !o.Check() || *enqueued != 2 { + t.Fatalf("changed input must enqueue: enqueued=%d", *enqueued) + } +} + +func TestRewriteWithOnlyANewTimestampIsNotAChange(t *testing.T) { + o, _, dir, enqueued := newObserver(t) + writeSnapshot(t, dir, withTimestamp(t, snapshotJSON(t, PhaseRunning, cand{id: "a"}), "2026-10-05T21:35:12Z")) + o.Check() + writeSnapshot(t, dir, withTimestamp(t, snapshotJSON(t, PhaseRunning, cand{id: "a"}), "2026-10-05T21:35:14Z")) + if o.Check() || *enqueued != 1 { + t.Fatalf("a new timestamp alone is not a semantic change: enqueued=%d", *enqueued) + } +} + +func TestInvalidSnapshotKeepsTheLastValidOneAndIsRecorded(t *testing.T) { + o, store, dir, enqueued := newObserver(t) + writeSnapshot(t, dir, snapshotJSON(t, PhaseRunning, cand{id: "a"})) + o.Check() + writeSnapshot(t, dir, []byte(`{"schemaVersion":2}`)) + if o.Check() || *enqueued != 1 { + t.Fatalf("an invalid snapshot must not enqueue: enqueued=%d", *enqueued) + } + in := store.Load() + if !errors.Is(in.Err, ErrProtocolViolation) || in.Snapshot == nil || in.Snapshot.Candidates[0].ID != "a" { + t.Fatalf("input = %+v", in) + } + writeSnapshot(t, dir, snapshotJSON(t, PhaseRunning, cand{id: "a"}, cand{id: "b"})) + if !o.Check() || store.Load().Err != nil { + t.Fatalf("a valid snapshot must clear the error: %+v", store.Load()) + } +} + +func TestRegressingSnapshotIsRejected(t *testing.T) { + for name, regressed := range map[string][2]int{"older round": {2, 99}, "fewer evaluated in the same round": {3, 4}} { + t.Run(name, func(t *testing.T) { + o, store, dir, enqueued := newObserver(t) + writeProgress(t, dir, PhaseRunning, 3, 5) + o.Check() + writeProgress(t, dir, PhaseRunning, regressed[0], regressed[1]) + if o.Check() || *enqueued != 1 { + t.Fatalf("a regressing snapshot must not be stored: enqueued=%d", *enqueued) + } + in := store.Load() + if !errors.Is(in.Err, ErrProtocolViolation) || ExitCode(in.Err) != ExitProtocolViolation { + t.Fatalf("err = %v", in.Err) + } + if in.Snapshot.Progress.Round != 3 || in.Snapshot.Progress.Evaluated != 5 { + t.Fatalf("the last valid snapshot must stay: %+v", in.Snapshot.Progress) + } + }) + } +} + +func TestTerminalSnapshotMayReportItsFinalEvaluatedTotal(t *testing.T) { + o, store, dir, _ := newObserver(t) + writeProgress(t, dir, PhaseRunning, 3, 9) + o.Check() + writeProgress(t, dir, PhaseSucceeded, 3, 7) + if !o.Check() || store.Load().Err != nil || !store.Load().Snapshot.Run.Terminal { + t.Fatalf("terminal snapshot rejected: %+v", store.Load()) + } +} + +func TestUnreadableSnapshotIsRecordedAsANonViolation(t *testing.T) { + o, store, dir, _ := newObserver(t) + if err := os.Mkdir(filepath.Join(dir, SnapshotFileName), 0o700); err != nil { // reading a directory fails + t.Fatal(err) + } + o.Check() + if err := store.Load().Err; err == nil || errors.Is(err, ErrProtocolViolation) { + t.Fatalf("err = %v, want a plain read error", err) + } +} + +func TestNextIntervalBacksOffWhileUnchangedAndResetsOnChange(t *testing.T) { + minimum, maximum := time.Second, 5*time.Second + got := []time.Duration{} + interval := minimum + for i := 0; i < 5; i++ { + interval = NextInterval(interval, minimum, maximum, false) + got = append(got, interval) + } + want := []time.Duration{2 * time.Second, 4 * time.Second, 5 * time.Second, 5 * time.Second, 5 * time.Second} + for i := range want { + if got[i] != want[i] { + t.Fatalf("intervals = %v, want %v", got, want) + } + } + if NextInterval(5*time.Second, minimum, maximum, true) != minimum { + t.Fatal("a change must reset the interval") + } +} + +func TestObserverRunEnqueuesAChangeAndStopsWithItsContext(t *testing.T) { + o, _, dir, _ := newObserver(t) + events := make(chan struct{}, 4) + o.Enqueue = func() { events <- struct{}{} } + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { done <- o.Run(ctx) }() + writeSnapshot(t, dir, snapshotJSON(t, PhaseRunning, cand{id: "a"})) + select { + case <-events: + case <-time.After(5 * time.Second): + t.Fatal("the observer never reported the snapshot") + } + cancel() + select { + case err := <-done: + if err != nil { + t.Fatalf("Run returned %v", err) + } + case <-time.After(5 * time.Second): + t.Fatal("Run did not stop") + } +} diff --git a/deploy/operator/internal/dgdrrunpublisher/kube.go b/deploy/operator/internal/dgdrrunpublisher/kube.go deleted file mode 100644 index 49887795dfb6..000000000000 --- a/deploy/operator/internal/dgdrrunpublisher/kube.go +++ /dev/null @@ -1,280 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package dgdrrunpublisher - -import ( - "context" - "encoding/json" - "fmt" - "strings" - - dgdv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1" - v1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" - "github.com/ai-dynamo/dynamo/deploy/operator/internal/dgdcreconcile" - v1beta2 "github.com/ai-dynamo/dynamo/deploy/operator/internal/dgdrrunpublisher/placeholderapi" - corev1 "k8s.io/api/core/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/api/meta" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/types" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" -) - -const ( - // LabelRunUID ties a candidate to one incarnation of its run. The run name can be - // reused after a delete and recreate; the UID cannot, so candidates left over from a - // previous run of the same name are never mistaken for current ones. - LabelRunUID = "nvidia.com/dgdr-run-uid" - // LabelCandidateID records the stable candidate id (the identity read back on list). - LabelCandidateID = "nvidia.com/dgdr-candidate-id" - - evaluatedCondition = "Evaluated" - - // AnnotationAdditionalResources carries the companion resources (for example - // generated ConfigMaps) rendered next to the DGD, as multi-document YAML. It is the - // same key and format the DGDR controller uses. - AnnotationAdditionalResources = "dgdr.nvidia.com/additional-resources" -) - -// KubeCluster is the Cluster implementation backed by a controller-runtime client. It -// should be a direct (uncached) client: the publisher is a short-lived sidecar, so it -// neither needs nor wants an informer cache. -// -// Required RBAC (namespaced, Role bound to the run's ServiceAccount): candidates -// get/list/create/delete and candidates/status update; runs get and runs/status patch; -// pods get (own pod, for Sweeper crash detection; resourceNames cannot be used because -// Job pod names are generated). -type KubeCluster struct { - Client client.Client - Namespace string - RunName string - PodName string - SweeperContainer string -} - -var _ Cluster = (*KubeCluster)(nil) - -func (k *KubeCluster) getRun(ctx context.Context) (*v1beta2.DynamoGraphDeploymentRun, error) { - var run v1beta2.DynamoGraphDeploymentRun - if err := k.Client.Get(ctx, types.NamespacedName{Namespace: k.Namespace, Name: k.RunName}, &run); err != nil { - return nil, fmt.Errorf("getting run: %w", err) - } - return &run, nil -} - -// RunUID is the UID of the current incarnation of the run; candidate names include it. -func (k *KubeCluster) RunUID(ctx context.Context) (string, error) { - run, err := k.getRun(ctx) - if err != nil { - return "", err - } - return string(run.UID), nil -} - -func (k *KubeCluster) ListCandidates(ctx context.Context) ([]dgdcreconcile.CurrentDGDC, error) { - run, err := k.getRun(ctx) - if err != nil { - return nil, err - } - var list v1beta2.DynamoGraphDeploymentCandidateList - if err := k.Client.List(ctx, &list, client.InNamespace(k.Namespace), client.MatchingLabels{LabelRunUID: string(run.UID)}); err != nil { - return nil, err - } - out := make([]dgdcreconcile.CurrentDGDC, 0, len(list.Items)) - for i := range list.Items { - out = append(out, dgdcreconcile.CurrentDGDC{ - Name: list.Items[i].Name, - ID: list.Items[i].Labels[LabelCandidateID], - Incomplete: !meta.IsStatusConditionTrue(list.Items[i].Status.Conditions, evaluatedCondition), - }) - } - return out, nil -} - -// decodeManifest extracts the DGD spec and the companion resources from a rendered -// manifest. A DGD in either supported apiVersion is converted to v1beta1, the same way -// the DGDR controller does it. -func decodeManifest(candidate dgdcreconcile.DesiredCandidate) (*v1beta1.DynamoGraphDeploymentSpec, string, error) { - manifest, err := ParseManifest(candidate.Spec) - if err != nil { - return nil, "", fmt.Errorf("candidate %s: %w", candidate.ID, err) - } - obj := &unstructured.Unstructured{Object: manifest.DGD} - var beta v1beta1.DynamoGraphDeployment - switch obj.GetAPIVersion() { - case dgdv1alpha1.GroupVersion.String(): - var alpha dgdv1alpha1.DynamoGraphDeployment - if err := runtime.DefaultUnstructuredConverter.FromUnstructured(obj.Object, &alpha); err != nil { - return nil, "", fmt.Errorf("candidate %s: %w", candidate.ID, err) - } - if err := alpha.ConvertTo(&beta); err != nil { - return nil, "", fmt.Errorf("candidate %s: %w", candidate.ID, err) - } - case v1beta1.GroupVersion.String(): - if err := runtime.DefaultUnstructuredConverter.FromUnstructured(obj.Object, &beta); err != nil { - return nil, "", fmt.Errorf("candidate %s: %w", candidate.ID, err) - } - default: - return nil, "", fmt.Errorf("candidate %s: unsupported DynamoGraphDeployment apiVersion %q", candidate.ID, obj.GetAPIVersion()) - } - return &beta.Spec, strings.Join(manifest.Companions, "\n---\n"), nil -} - -// CreateCandidate creates the candidate and populates its status exactly once. Both -// steps are idempotent so a retry after a partial failure converges; the publisher -// calls it again for a candidate whose status was never populated. -func (k *KubeCluster) CreateCandidate(ctx context.Context, name string, candidate dgdcreconcile.DesiredCandidate) error { - spec, companions, err := decodeManifest(candidate) - if err != nil { - return err - } - run, err := k.getRun(ctx) - if err != nil { - return err - } - parameters, err := rawExtension(candidate.Parameters) - if err != nil { - return err - } - object := &v1beta2.DynamoGraphDeploymentCandidate{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: k.Namespace, - Name: name, - Labels: map[string]string{LabelRunUID: string(run.UID), LabelCandidateID: candidate.ID}, - }, - Spec: v1beta2.DynamoGraphDeploymentCandidateSpec{ - DynamoGraphDeploymentSpec: *spec, - Parameters: parameters, - }, - } - if companions != "" { - object.Annotations = map[string]string{AnnotationAdditionalResources: companions} - } - if err := controllerutil.SetControllerReference(run, object, k.Client.Scheme()); err != nil { - return err - } - err = k.Client.Create(ctx, object) - if apierrors.IsAlreadyExists(err) { - err = k.replaceIfIncomplete(ctx, object, run) - } - if err != nil { - return err - } - - var stored v1beta2.DynamoGraphDeploymentCandidate - if err := k.Client.Get(ctx, types.NamespacedName{Namespace: k.Namespace, Name: name}, &stored); err != nil { - return err - } - if meta.IsStatusConditionTrue(stored.Status.Conditions, evaluatedCondition) { - return nil // status already populated; candidates are immutable afterwards - } - metrics, err := rawExtension(candidate.Metrics) - if err != nil { - return err - } - stored.Status.Metrics = metrics - meta.SetStatusCondition(&stored.Status.Conditions, metav1.Condition{ - Type: evaluatedCondition, - Status: metav1.ConditionTrue, - Reason: evaluatedCondition, - Message: "evaluated by the Sweeper", - }) - return k.Client.Status().Update(ctx, &stored) -} - -// replaceIfIncomplete handles a candidate that already exists. One controlled by another -// run is an error. A complete one is left untouched (candidates are immutable). One whose -// status was never written may hold the spec of an earlier snapshot that reused the id, -// so it is deleted and created again from the current manifest. -func (k *KubeCluster) replaceIfIncomplete(ctx context.Context, object *v1beta2.DynamoGraphDeploymentCandidate, run *v1beta2.DynamoGraphDeploymentRun) error { - var stored v1beta2.DynamoGraphDeploymentCandidate - if err := k.Client.Get(ctx, types.NamespacedName{Namespace: k.Namespace, Name: object.Name}, &stored); err != nil { - return err - } - if !metav1.IsControlledBy(&stored, run) { - return fmt.Errorf("candidate %s already exists and is not controlled by run %s (uid %s)", object.Name, k.RunName, run.UID) - } - if meta.IsStatusConditionTrue(stored.Status.Conditions, evaluatedCondition) { - return nil - } - uid := stored.UID - if err := k.Client.Delete(ctx, &stored, client.Preconditions{UID: &uid}); client.IgnoreNotFound(err) != nil { - return err - } - if err := k.Client.Create(ctx, object.DeepCopy()); err != nil { - return fmt.Errorf("recreating incomplete candidate %s: %w", object.Name, err) - } - return nil -} - -func rawExtension(value map[string]any) (*runtime.RawExtension, error) { - if value == nil { - value = map[string]any{} - } - raw, err := json.Marshal(value) - if err != nil { - return nil, err - } - return &runtime.RawExtension{Raw: raw}, nil -} - -func (k *KubeCluster) DeleteCandidate(ctx context.Context, name string) error { - object := &v1beta2.DynamoGraphDeploymentCandidate{ObjectMeta: metav1.ObjectMeta{Namespace: k.Namespace, Name: name}} - return client.IgnoreNotFound(k.Client.Delete(ctx, object)) -} - -func (k *KubeCluster) PatchRunStatus(ctx context.Context, status RunStatus) error { - var run v1beta2.DynamoGraphDeploymentRun - if err := k.Client.Get(ctx, types.NamespacedName{Namespace: k.Namespace, Name: k.RunName}, &run); err != nil { - return err - } - base := run.DeepCopy() - run.Status.Message = status.Message - run.Status.Progress = &v1beta2.RunProgress{Round: status.Round, Evaluated: status.Evaluated} - if !status.LastProgress.IsZero() { - progressed := metav1.NewTime(status.LastProgress) - run.Status.LastProgressTime = &progressed - } - refs := make([]v1beta2.CandidateRef, 0, len(status.CandidateNames)) - for _, name := range status.CandidateNames { - refs = append(refs, v1beta2.CandidateRef{Name: name}) - } - run.Status.CandidateRefs = refs - return k.Client.Status().Patch(ctx, &run, client.MergeFrom(base)) -} - -// SweeperState reads the Sweeper container's status from this pod's own status. -func (k *KubeCluster) SweeperState(ctx context.Context) (SweeperState, error) { - var pod corev1.Pod - if err := k.Client.Get(ctx, types.NamespacedName{Namespace: k.Namespace, Name: k.PodName}, &pod); err != nil { - return SweeperState{}, err - } - for _, status := range pod.Status.ContainerStatuses { - if status.Name != k.SweeperContainer { - continue - } - if terminated := status.State.Terminated; terminated != nil { - return SweeperState{Exited: true, ExitCode: terminated.ExitCode}, nil - } - return SweeperState{}, nil - } - return SweeperState{}, nil -} diff --git a/deploy/operator/internal/dgdrrunpublisher/manager.go b/deploy/operator/internal/dgdrrunpublisher/manager.go new file mode 100644 index 000000000000..2ca8d2c04b84 --- /dev/null +++ b/deploy/operator/internal/dgdrrunpublisher/manager.go @@ -0,0 +1,208 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dgdrrunpublisher + +import ( + "context" + "errors" + "fmt" + "path/filepath" + "time" + + v1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" + v1beta2 "github.com/ai-dynamo/dynamo/deploy/operator/internal/dgdrrunpublisher/placeholderapi" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/cache" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/manager" + metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/controller-runtime/pkg/source" +) + +// Config is everything one publisher needs to know about its run and pod. +type Config struct { + Namespace string + RunName string + // RunUID is the UID of the run this pod serves. The run controller knows it when it + // builds the Job, which is what lets the cache select this run's candidates by label. + RunUID string + PodName string + SweeperContainer string + SnapshotDir string + // MinPollInterval and MaxPollInterval bound the snapshot check interval, which backs + // off from the minimum to the maximum while the input is unchanged. + MinPollInterval time.Duration + MaxPollInterval time.Duration +} + +func (c Config) validate() error { + switch { + case c.Namespace == "", c.RunName == "", c.RunUID == "", c.PodName == "", c.SnapshotDir == "": + return errors.New("namespace, run name, run uid, pod name and snapshot dir are required") + case c.MinPollInterval <= 0 || c.MaxPollInterval < c.MinPollInterval: + return errors.New("poll intervals must be positive and the maximum at least the minimum") + } + return nil +} + +// NewScheme registers every type the publisher reads or writes. +func NewScheme() (*runtime.Scheme, error) { + scheme := runtime.NewScheme() + for _, add := range []func(*runtime.Scheme) error{clientgoscheme.AddToScheme, v1beta1.AddToScheme, v1beta2.AddToScheme} { + if err := add(scheme); err != nil { + return nil, err + } + } + return scheme, nil +} + +// cacheOptions limits the manager's informers to this run, its candidates and its own +// pod: one namespace, plus a field selector on the names of the run and the pod and a +// label selector on the run UID for the candidates. +func cacheOptions(c Config) cache.Options { + return cache.Options{ + DefaultNamespaces: map[string]cache.Config{c.Namespace: {}}, + ByObject: map[client.Object]cache.ByObject{ + &v1beta2.DynamoGraphDeploymentRun{}: { + Field: fields.OneTermEqualSelector("metadata.name", c.RunName), + }, + &v1beta2.DynamoGraphDeploymentCandidate{}: { + Label: labels.SelectorFromSet(labels.Set{LabelRunUID: c.RunUID}), + }, + &corev1.Pod{}: { + Field: fields.OneTermEqualSelector("metadata.name", c.PodName), + }, + }, + } +} + +// Run starts the controller manager and returns the publisher's outcome (see the package +// comment for its meaning and ExitCode for the exit code), or ctx's error if the process +// is told to stop first. +func Run(ctx context.Context, restConfig *rest.Config, cfg Config) error { + if err := cfg.validate(); err != nil { + return err + } + scheme, err := NewScheme() + if err != nil { + return fmt.Errorf("building scheme: %w", err) + } + shutdownTimeout := 5 * time.Second + mgr, err := ctrl.NewManager(restConfig, ctrl.Options{ + Scheme: scheme, + Cache: cacheOptions(cfg), + Metrics: metricsserver.Options{BindAddress: "0"}, + LeaderElection: false, + GracefulShutdownTimeout: &shutdownTimeout, + }) + if err != nil { + return fmt.Errorf("creating manager: %w", err) + } + + key := types.NamespacedName{Namespace: cfg.Namespace, Name: cfg.RunName} + events := make(chan event.GenericEvent, 1) + store := &InputStore{} + observer := &Observer{ + Store: store, + Path: SnapshotPath(cfg.SnapshotDir), + MinInterval: cfg.MinPollInterval, + MaxInterval: cfg.MaxPollInterval, + Enqueue: func() { + // One pending event is enough: the workqueue deduplicates by key anyway. + select { + case events <- event.GenericEvent{Object: runKeyObject(key)}: + default: + } + }, + } + completion := NewCompletion() + reconciler := &Reconciler{ + Client: mgr.GetClient(), + Namespace: cfg.Namespace, + RunName: cfg.RunName, + RunUID: cfg.RunUID, + PodName: cfg.PodName, + SweeperContainer: cfg.SweeperContainer, + Observer: observer, + Store: store, + Completion: completion, + } + + podToRun := handler.EnqueueRequestsFromMapFunc(func(context.Context, client.Object) []reconcile.Request { + return []reconcile.Request{{NamespacedName: key}} + }) + if err := ctrl.NewControllerManagedBy(mgr). + Named("dgdr-run-publisher"). + For(&v1beta2.DynamoGraphDeploymentRun{}). + Owns(&v1beta2.DynamoGraphDeploymentCandidate{}). + Watches(&corev1.Pod{}, podToRun). + WatchesRawSource(source.Channel(events, &handler.EnqueueRequestForObject{})). + WithOptions(controller.Options{MaxConcurrentReconciles: 1}). + Complete(reconciler); err != nil { + return fmt.Errorf("building controller: %w", err) + } + if err := mgr.Add(manager.RunnableFunc(observer.Run)); err != nil { + return fmt.Errorf("adding input observer: %w", err) + } + + runCtx, cancel := context.WithCancel(ctx) + defer cancel() + go func() { + select { + case <-completion.Done(): + cancel() // the outcome is known: stop the manager and exit + case <-runCtx.Done(): + } + }() + startErr := mgr.Start(runCtx) + + if completion.Finished() { + return completion.Err() + } + if ctx.Err() != nil { + return ctx.Err() + } + if startErr != nil { + return fmt.Errorf("manager stopped: %w", startErr) + } + return errors.New("manager stopped before the run finished") +} + +// SnapshotPath is where the Sweeper writes the snapshot inside the shared directory. +func SnapshotPath(dir string) string { + return filepath.Join(dir, SnapshotFileName) +} + +// runKeyObject is the object carried by enqueue events; only its name and namespace +// matter to the handler. +func runKeyObject(key types.NamespacedName) client.Object { + run := &v1beta2.DynamoGraphDeploymentRun{} + run.Namespace, run.Name = key.Namespace, key.Name + return run +} diff --git a/deploy/operator/internal/dgdrrunpublisher/outcome.go b/deploy/operator/internal/dgdrrunpublisher/outcome.go new file mode 100644 index 000000000000..aea5b707ebee --- /dev/null +++ b/deploy/operator/internal/dgdrrunpublisher/outcome.go @@ -0,0 +1,114 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dgdrrunpublisher + +import ( + "errors" + "fmt" + "sync" +) + +// Process exit codes of the publisher binary. +const ( + ExitAcknowledged = 0 + ExitReconcileFailed = 1 + ExitProtocolViolation = 3 + ExitRunFailed = 4 +) + +// ErrMissingTerminalSnapshot means the Sweeper exited successfully without publishing a +// terminal snapshot, which violates the producer protocol. +var ErrMissingTerminalSnapshot = fmt.Errorf("%w: sweeper exited without publishing a terminal snapshot", ErrProtocolViolation) + +// ErrRunFailed means the Sweeper published a terminal Failed snapshot. The snapshot is +// reconciled first; the error keeps a producer that reports failure but exits 0 from +// letting the Job complete successfully. +var ErrRunFailed = errors.New("sweeper reported a failed run") + +// ExitCode maps the result of Run to the process exit code. +func ExitCode(err error) int { + switch { + case err == nil: + return ExitAcknowledged + case errors.Is(err, ErrProtocolViolation): + return ExitProtocolViolation + case errors.Is(err, ErrRunFailed): + return ExitRunFailed + default: + return ExitReconcileFailed + } +} + +// SweeperState is the observed state of the Sweeper container in this pod. +type SweeperState struct { + Exited bool + ExitCode int32 +} + +// terminalResult is the outcome once a terminal snapshot has been reconciled. +func terminalResult(phase string) error { + if phase == PhaseFailed { + return ErrRunFailed + } + return nil +} + +// exitResult is the outcome when the Sweeper exited and the last snapshot it wrote (if +// any) has been reconciled, but that snapshot is not terminal. +func exitResult(state SweeperState) error { + if state.ExitCode != 0 { + // Crash or caught failure without a terminal snapshot: the last-known projection + // is reconciled; the Sweeper's own exit code fails the Job. + return nil + } + return ErrMissingTerminalSnapshot +} + +// Completion records the publisher's outcome exactly once and wakes whoever waits on it. +type Completion struct { + once sync.Once + done chan struct{} + err error +} + +// NewCompletion returns an unfinished Completion. +func NewCompletion() *Completion { return &Completion{done: make(chan struct{})} } + +// Finish records err as the outcome. Only the first call has an effect. +func (c *Completion) Finish(err error) { + c.once.Do(func() { + c.err = err + close(c.done) + }) +} + +// Done is closed once the outcome is known. +func (c *Completion) Done() <-chan struct{} { return c.done } + +// Err is the outcome. It is only meaningful after Done is closed. +func (c *Completion) Err() error { return c.err } + +// Finished reports whether the outcome is known. +func (c *Completion) Finished() bool { + select { + case <-c.done: + return true + default: + return false + } +} diff --git a/deploy/operator/internal/dgdrrunpublisher/outcome_test.go b/deploy/operator/internal/dgdrrunpublisher/outcome_test.go new file mode 100644 index 000000000000..7aa0d89a4468 --- /dev/null +++ b/deploy/operator/internal/dgdrrunpublisher/outcome_test.go @@ -0,0 +1,78 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dgdrrunpublisher + +import ( + "errors" + "fmt" + "testing" +) + +func TestExitCodes(t *testing.T) { + cases := []struct { + err error + want int + }{ + {nil, ExitAcknowledged}, + {ErrRunFailed, ExitRunFailed}, + {ErrMissingTerminalSnapshot, ExitProtocolViolation}, + {violation("bad"), ExitProtocolViolation}, + {fmt.Errorf("reconciling last snapshot: %w", violation("bad")), ExitProtocolViolation}, + {errors.New("api down"), ExitReconcileFailed}, + } + for _, tc := range cases { + if got := ExitCode(tc.err); got != tc.want { + t.Errorf("ExitCode(%v) = %d, want %d", tc.err, got, tc.want) + } + } +} + +func TestTerminalResult(t *testing.T) { + if err := terminalResult(PhaseSucceeded); err != nil { + t.Fatalf("succeeded: %v", err) + } + if err := terminalResult(PhaseFailed); !errors.Is(err, ErrRunFailed) { + t.Fatalf("failed: %v", err) + } +} + +func TestExitResultWithoutATerminalSnapshot(t *testing.T) { + if err := exitResult(SweeperState{Exited: true, ExitCode: 137}); err != nil { + t.Fatalf("a crash is reported by the Sweeper's own exit code, not the publisher: %v", err) + } + if err := exitResult(SweeperState{Exited: true}); !errors.Is(err, ErrMissingTerminalSnapshot) { + t.Fatalf("a clean exit without a terminal snapshot is a protocol violation: %v", err) + } +} + +func TestCompletionKeepsTheFirstOutcome(t *testing.T) { + c := NewCompletion() + if c.Finished() { + t.Fatal("finished before Finish") + } + c.Finish(ErrRunFailed) + c.Finish(nil) + select { + case <-c.Done(): + default: + t.Fatal("Done not closed") + } + if !c.Finished() || !errors.Is(c.Err(), ErrRunFailed) { + t.Fatalf("err = %v", c.Err()) + } +} diff --git a/deploy/operator/internal/dgdrrunpublisher/plan.go b/deploy/operator/internal/dgdrrunpublisher/plan.go new file mode 100644 index 000000000000..ff515b677bed --- /dev/null +++ b/deploy/operator/internal/dgdrrunpublisher/plan.go @@ -0,0 +1,161 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dgdrrunpublisher + +import ( + "crypto/sha256" + "encoding/hex" + "slices" + "strconv" + "strings" + "time" + + "github.com/ai-dynamo/dynamo/deploy/operator/internal/dgdcreconcile" +) + +// SnapshotFileName is the file the Sweeper container writes. +const SnapshotFileName = "dgdr_run_snapshot.yaml" + +// maxObjectName is the Kubernetes limit on an object name (DNS subdomain). +const maxObjectName = 253 + +// RunStatus is the part of DynamoGraphDeploymentRun.status the publisher owns: the +// ordered candidate references plus progress. CandidateNames is in rank order (best +// first for scalar searches). LastProgress is the zero time when the snapshot carries +// no usable timestamp, in which case the existing value is left untouched. +type RunStatus struct { + Message string + HasProgress bool + Round int32 + Evaluated int32 + LastProgress time.Time + CandidateNames []string +} + +// DesiredState is what one snapshot says should exist in the cluster. +type DesiredState struct { + // Candidates are the materialized candidates, in snapshot (rank) order. + Candidates []dgdcreconcile.DesiredCandidate + // Names maps a candidate id to its DGDC name. + Names map[string]string + Status RunStatus + Terminal bool + Phase string +} + +// CandidateName is the DGDC name for a candidate id. It depends only on the run +// incarnation (name and UID) and the stable id, never on rank. The UID component keeps a +// recreated run of the same name from colliding with candidates of the previous one that +// are still waiting for garbage collection. A run name too long to leave room for the +// rest is truncated and a hash of the full run name is added, so names stay unique and +// within the limit. +func CandidateName(runName, runUID, id string) string { + uidSum := sha256.Sum256([]byte(runUID)) + suffix := "-" + hex.EncodeToString(uidSum[:4]) + "-" + id + if len(runName)+len(suffix) <= maxObjectName { + return runName + suffix + } + nameSum := sha256.Sum256([]byte(runName)) + hash := hex.EncodeToString(nameSum[:6]) + room := maxObjectName - len(suffix) - len(hash) - 1 + return strings.TrimRight(runName[:room], "-.") + "-" + hash + suffix +} + +// Desire projects a snapshot onto the desired cluster state of one run incarnation. +// Candidates whose materialization failed are not created; they are only counted in the +// run message. +func Desire(runName, runUID string, snap *Snapshot) DesiredState { + state := DesiredState{ + Names: make(map[string]string, len(snap.Candidates)), + Terminal: snap.Run.Terminal, + Phase: snap.Run.Phase, + } + failed := 0 + var names []string + for _, candidate := range snap.Candidates { + if candidate.Outcome != OutcomeMaterialized { + failed++ + continue + } + name := CandidateName(runName, runUID, candidate.ID) + state.Names[candidate.ID] = name + names = append(names, name) + state.Candidates = append(state.Candidates, dgdcreconcile.DesiredCandidate{ + ID: candidate.ID, + Spec: candidate.Manifest, + Parameters: candidate.Parameters, + Metrics: candidate.Metrics, + }) + } + state.Status = RunStatus{ + Message: runMessage(snap, failed), + HasProgress: true, + Round: snap.Progress.Round, + Evaluated: snap.Progress.Evaluated, + LastProgress: progressTime(snap), + CandidateNames: names, + } + return state +} + +func runMessage(snap *Snapshot, failed int) string { + message := snap.Run.Message + if snap.Run.Error != "" { + message = snap.Run.Error + } + if failed > 0 { + suffix := fmtFailed(failed) + if message == "" { + return suffix + } + return message + "; " + suffix + } + return message +} + +func fmtFailed(failed int) string { + return strconv.Itoa(failed) + " candidate(s) failed materialization" +} + +// progressTime is the snapshot's own timestamp (written by the Sweeper container in the +// same pod) truncated to the second the API stores, or the zero time when it is missing +// or malformed. Using the snapshot's time instead of the clock keeps reconciliation +// deterministic. +func progressTime(snap *Snapshot) time.Time { + parsed, err := time.Parse(time.RFC3339, snap.Timestamp) + if err != nil { + return time.Time{} + } + return parsed.UTC().Truncate(time.Second) +} + +// NeedsUpdate reports whether the observed run status differs from the desired one, so +// the status is only written when something changed. A zero desired LastProgress means +// "leave the observed value alone". +func (desired RunStatus) NeedsUpdate(observed RunStatus) bool { + if desired.Message != observed.Message || desired.HasProgress != observed.HasProgress { + return true + } + if desired.HasProgress && (desired.Round != observed.Round || desired.Evaluated != observed.Evaluated) { + return true + } + if !desired.LastProgress.IsZero() && !desired.LastProgress.Equal(observed.LastProgress) { + return true + } + return !slices.Equal(desired.CandidateNames, observed.CandidateNames) +} diff --git a/deploy/operator/internal/dgdrrunpublisher/plan_test.go b/deploy/operator/internal/dgdrrunpublisher/plan_test.go new file mode 100644 index 000000000000..a1f05e946d06 --- /dev/null +++ b/deploy/operator/internal/dgdrrunpublisher/plan_test.go @@ -0,0 +1,123 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dgdrrunpublisher + +import ( + "reflect" + "strings" + "testing" + "time" +) + +func TestCandidateNameIsBoundedAndUniquePerRunIncarnation(t *testing.T) { + id := "evaluated-point-0123456789ab" + if cn("a") != CandidateName("run", testRunUID, "a") || cn("a") == CandidateName("run", "uid-2", "a") { + t.Fatalf("a recreated run must get different names: %q", cn("a")) + } + long := strings.Repeat("r", 253) + other := strings.Repeat("r", 252) + "s" + name := CandidateName(long, testRunUID, id) + if len(name) > 253 || !strings.HasSuffix(name, "-"+id) || name == CandidateName(other, testRunUID, id) || name != CandidateName(long, testRunUID, id) { + t.Fatalf("name = %q (%d)", name, len(name)) + } +} + +func desire(t *testing.T, data []byte) DesiredState { + t.Helper() + return Desire("run", testRunUID, mustParse(t, data)) +} + +func TestSwapKeepsCandidateNamesAndOnlyReordersReferences(t *testing.T) { + first := desire(t, snapshotJSON(t, PhaseRunning, cand{id: "a"}, cand{id: "b"})) + second := desire(t, snapshotJSON(t, PhaseRunning, cand{id: "b"}, cand{id: "a"})) + if !reflect.DeepEqual(first.Names, second.Names) { + t.Fatalf("names depend on rank: %v vs %v", first.Names, second.Names) + } + if want := []string{cn("b"), cn("a")}; !reflect.DeepEqual(second.Status.CandidateNames, want) { + t.Fatalf("ordered references = %v, want %v", second.Status.CandidateNames, want) + } + if !second.Status.NeedsUpdate(first.Status) { + t.Fatal("a reorder must update the run's references") + } +} + +func TestFailedMaterializationIsNotDesiredButCounted(t *testing.T) { + state := desire(t, snapshotJSON(t, PhaseRunning, cand{id: "a"}, cand{id: "x", failed: true})) + if len(state.Candidates) != 1 || state.Candidates[0].ID != "a" { + t.Fatalf("candidates = %+v", state.Candidates) + } + if !reflect.DeepEqual(state.Status.CandidateNames, []string{cn("a")}) { + t.Fatalf("refs must not include the failed candidate: %v", state.Status.CandidateNames) + } + if !strings.Contains(state.Status.Message, "1 candidate(s) failed materialization") { + t.Fatalf("message = %q", state.Status.Message) + } +} + +func TestRunErrorReplacesTheMessage(t *testing.T) { + snap := mustParse(t, snapshotJSON(t, PhaseFailed, cand{id: "a"})) + snap.Run.Message, snap.Run.Error = "searching", "out of memory" + if got := Desire("run", testRunUID, snap).Status.Message; got != "out of memory" { + t.Fatalf("message = %q", got) + } +} + +func TestLastProgressComesFromTheSnapshotTimestamp(t *testing.T) { + data := withTimestamp(t, snapshotJSON(t, PhaseRunning, cand{id: "a"}), "2026-10-05T21:35:12Z") + want := time.Date(2026, 10, 5, 21, 35, 12, 0, time.UTC) + if got := desire(t, data).Status.LastProgress; !got.Equal(want) { + t.Fatalf("LastProgress = %v, want %v", got, want) + } +} + +func TestMissingOrMalformedTimestampLeavesProgressTimeAlone(t *testing.T) { + for _, timestamp := range []string{"", "yesterday"} { + state := desire(t, withTimestamp(t, snapshotJSON(t, PhaseRunning, cand{id: "a"}), timestamp)) + if !state.Status.LastProgress.IsZero() { + t.Fatalf("timestamp %q: LastProgress = %v, want zero", timestamp, state.Status.LastProgress) + } + observed := state.Status + observed.LastProgress = time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + if state.Status.NeedsUpdate(observed) { + t.Fatalf("timestamp %q: an unknown progress time must not force an update", timestamp) + } + } +} + +func TestNeedsUpdateOnlyWhenSomethingDiffers(t *testing.T) { + base := desire(t, withTimestamp(t, snapshotJSON(t, PhaseRunning, cand{id: "a"}), "2026-10-05T21:35:12Z")).Status + if base.NeedsUpdate(base) { + t.Fatal("identical status needs no write") + } + changes := map[string]func(*RunStatus){ + "message": func(s *RunStatus) { s.Message = "other" }, + "round": func(s *RunStatus) { s.Round++ }, + "evaluated": func(s *RunStatus) { s.Evaluated++ }, + "no progress": func(s *RunStatus) { s.HasProgress = false }, + "progress time": func(s *RunStatus) { s.LastProgress = s.LastProgress.Add(time.Second) }, + "references": func(s *RunStatus) { s.CandidateNames = nil }, + } + for name, mutate := range changes { + observed := base + observed.CandidateNames = append([]string(nil), base.CandidateNames...) + mutate(&observed) + if !base.NeedsUpdate(observed) { + t.Errorf("%s: a difference must trigger an update", name) + } + } +} diff --git a/deploy/operator/internal/dgdrrunpublisher/publisher.go b/deploy/operator/internal/dgdrrunpublisher/publisher.go deleted file mode 100644 index 500d05f1f6b4..000000000000 --- a/deploy/operator/internal/dgdrrunpublisher/publisher.go +++ /dev/null @@ -1,378 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// Package dgdrrunpublisher is the sidecar that turns the Sweeper's pod-local desired -// state snapshot into Kubernetes state: it creates/deletes DynamoGraphDeploymentCandidates -// and patches the ordered candidate references and progress on -// DynamoGraphDeploymentRun.status. -// -// It runs as a regular container of the same Job pod as the Sweeper, sharing an -// emptyDir with it. There is no ConfigMap and no cross-pod channel: the file is the -// interface. Because the publisher is a regular container, the Job only completes once -// the publisher has exited, so its successful exit is the acknowledgement that the final -// snapshot was reconciled. The run controller derives the terminal Completed condition -// from the Job result; the publisher never writes a phase. -// -// Termination state machine (see Publisher.Run): -// -// terminal Succeeded snapshot reconciled -> exit 0 -// terminal Failed snapshot reconciled -> exit ExitRunFailed -// Sweeper exited non-zero, no terminal snapshot -> reconcile last snapshot, exit 0 -// (the Sweeper's exit fails the Job) -// Sweeper exited zero, no terminal snapshot -> reconcile last snapshot, -// exit ExitProtocolViolation -// invalid or regressing snapshot, Sweeper exited -> exit ExitProtocolViolation -// final/last snapshot cannot be reconciled -> exit ExitReconcileFailed -package dgdrrunpublisher - -import ( - "context" - "crypto/sha256" - "encoding/hex" - "errors" - "fmt" - "log" - "os" - "path/filepath" - "reflect" - "strings" - "time" - - "github.com/ai-dynamo/dynamo/deploy/operator/internal/dgdcreconcile" -) - -// Process exit codes of the publisher binary. -const ( - ExitAcknowledged = 0 - ExitReconcileFailed = 1 - ExitProtocolViolation = 3 - ExitRunFailed = 4 -) - -// ErrMissingTerminalSnapshot means the Sweeper exited successfully without publishing a -// terminal snapshot, which violates the producer protocol. -var ErrMissingTerminalSnapshot = fmt.Errorf("%w: sweeper exited without publishing a terminal snapshot", ErrProtocolViolation) - -// ErrRunFailed means the Sweeper published a terminal Failed snapshot. The snapshot is -// reconciled first; the error keeps a producer that reports failure but exits 0 from -// letting the Job complete successfully. -var ErrRunFailed = errors.New("sweeper reported a failed run") - -// ExitCode maps the result of Run to the process exit code. -func ExitCode(err error) int { - switch { - case err == nil: - return ExitAcknowledged - case errors.Is(err, ErrProtocolViolation): - return ExitProtocolViolation - case errors.Is(err, ErrRunFailed): - return ExitRunFailed - default: - return ExitReconcileFailed - } -} - -// SnapshotFileName is the file the Sweeper container writes. -const SnapshotFileName = "dgdr_run_snapshot.yaml" - -// RunStatus is the part of DynamoGraphDeploymentRun.status the publisher owns: the -// ordered candidate references plus progress. CandidateNames is in rank order (best -// first for scalar searches). LastProgress is the zero time when the snapshot carries -// no usable timestamp, in which case the existing value is left untouched. -type RunStatus struct { - Message string - Round int32 - Evaluated int32 - LastProgress time.Time - CandidateNames []string -} - -// SweeperState is the observed state of the Sweeper container in this pod. -type SweeperState struct { - Exited bool - ExitCode int32 -} - -// Cluster is everything the publisher needs from Kubernetes. -type Cluster interface { - ListCandidates(ctx context.Context) ([]dgdcreconcile.CurrentDGDC, error) - // CreateCandidate must be idempotent: an already-existing candidate is success. - CreateCandidate(ctx context.Context, name string, candidate dgdcreconcile.DesiredCandidate) error - DeleteCandidate(ctx context.Context, name string) error - PatchRunStatus(ctx context.Context, status RunStatus) error - // RunUID identifies the current incarnation of the run, which candidate names include. - RunUID(ctx context.Context) (string, error) - // SweeperState reads the Sweeper container's status from this pod. - SweeperState(ctx context.Context) (SweeperState, error) -} - -type Publisher struct { - Cluster Cluster - SnapshotDir string - RunName string - PollInterval time.Duration - - lastStatus *RunStatus - haveSnapshot bool // a snapshot has been reconciled - round int32 - evaluated int32 - terminal bool // the reconciled snapshot was terminal - phase string -} - -// maxObjectName is the Kubernetes limit on an object name (DNS subdomain). -const maxObjectName = 253 - -// CandidateName is the DGDC name for a candidate id. It depends only on the run -// incarnation (name and UID) and the stable id, never on rank. The UID component keeps a -// recreated run of the same name from colliding with candidates of the previous one that -// are still waiting for garbage collection. A run name too long to leave room for the -// rest is truncated and a hash of the full run name is added, so names stay unique and -// within the limit. -func CandidateName(runName, runUID, id string) string { - uidSum := sha256.Sum256([]byte(runUID)) - suffix := "-" + hex.EncodeToString(uidSum[:4]) + "-" + id - if len(runName)+len(suffix) <= maxObjectName { - return runName + suffix - } - nameSum := sha256.Sum256([]byte(runName)) - hash := hex.EncodeToString(nameSum[:6]) - room := maxObjectName - len(suffix) - len(hash) - 1 - return strings.TrimRight(runName[:room], "-.") + "-" + hash + suffix -} - -// Reconcile applies one snapshot: create missing candidates, patch the run status -// (ordered references and progress), then delete obsolete candidates. Existing -// candidates are never updated. It is idempotent and safe to retry. -func (p *Publisher) Reconcile(ctx context.Context, snap *Snapshot) error { - runUID, err := p.Cluster.RunUID(ctx) - if err != nil { - return fmt.Errorf("reading run: %w", err) - } - var desired []dgdcreconcile.DesiredCandidate - var names []string - failed := 0 - for _, candidate := range snap.Candidates { - if candidate.Outcome != OutcomeMaterialized { - failed++ - continue - } - desired = append(desired, dgdcreconcile.DesiredCandidate{ - ID: candidate.ID, - Spec: candidate.Manifest, - Parameters: candidate.Parameters, - Metrics: candidate.Metrics, - }) - names = append(names, CandidateName(p.RunName, runUID, candidate.ID)) - } - - current, err := p.Cluster.ListCandidates(ctx) - if err != nil { - return fmt.Errorf("listing candidates: %w", err) - } - actions, err := dgdcreconcile.ComputeActions(desired, current) - if err != nil { - return err - } - for _, candidate := range actions.Creates { - if err := p.Cluster.CreateCandidate(ctx, CandidateName(p.RunName, runUID, candidate.ID), candidate); err != nil { - return fmt.Errorf("creating candidate %s: %w", candidate.ID, err) - } - } - - status := RunStatus{ - Message: runMessage(snap, failed), - Round: snap.Progress.Round, - Evaluated: snap.Progress.Evaluated, - LastProgress: progressTime(snap), - CandidateNames: names, - } - if err := p.patchStatus(ctx, status); err != nil { - return err - } - - for _, name := range actions.Deletes { - if err := p.Cluster.DeleteCandidate(ctx, name); err != nil { - return fmt.Errorf("deleting candidate %s: %w", name, err) - } - } - return nil -} - -func runMessage(snap *Snapshot, failed int) string { - message := snap.Run.Message - if snap.Run.Error != "" { - message = snap.Run.Error - } - if failed > 0 { - suffix := fmt.Sprintf("%d candidate(s) failed materialization", failed) - if message == "" { - return suffix - } - return message + "; " + suffix - } - return message -} - -// progressTime is the snapshot's own timestamp (written by the Sweeper container in the -// same pod), or the zero time when it is missing or malformed. Using the snapshot's time -// instead of the clock keeps reconciliation deterministic. -func progressTime(snap *Snapshot) time.Time { - parsed, err := time.Parse(time.RFC3339, snap.Timestamp) - if err != nil { - return time.Time{} - } - return parsed.UTC() -} - -// patchStatus skips the API call when nothing changed since the last successful patch, -// so repeated identical snapshots cause no writes. -func (p *Publisher) patchStatus(ctx context.Context, status RunStatus) error { - if p.lastStatus != nil && reflect.DeepEqual(*p.lastStatus, status) { - return nil - } - if err := p.Cluster.PatchRunStatus(ctx, status); err != nil { - return fmt.Errorf("patching run status: %w", err) - } - p.lastStatus = &status - return nil -} - -// syncOnce reads the snapshot file and reconciles it. A missing file is not an error. -// An unchanged snapshot is reconciled again on purpose: the diff makes that free when -// the cluster matches, and repairs it (for example a deleted candidate) when it does not. -func (p *Publisher) syncOnce(ctx context.Context) error { - data, err := os.ReadFile(filepath.Join(p.SnapshotDir, SnapshotFileName)) - if errors.Is(err, os.ErrNotExist) { - return nil - } - if err != nil { - return err - } - snap, err := ParseSnapshot(data) - if err != nil { - return err - } - if err := p.checkProgress(snap); err != nil { - return err - } - if err := p.Reconcile(ctx, snap); err != nil { - return err - } - p.haveSnapshot = true - p.round = snap.Progress.Round - p.evaluated = snap.Progress.Evaluated - p.terminal = snap.Run.Terminal - p.phase = snap.Run.Phase - return nil -} - -// checkProgress rejects a snapshot that moves progress backwards: an older round, or -// fewer evaluated points within the same round. A terminal snapshot is exempt from the -// evaluated check because its count is the search's final total, not the live counter. -func (p *Publisher) checkProgress(snap *Snapshot) error { - if !p.haveSnapshot { - return nil - } - if snap.Progress.Round < p.round { - return violation("snapshot round regressed from %d to %d", p.round, snap.Progress.Round) - } - if !snap.Run.Terminal && snap.Progress.Round == p.round && snap.Progress.Evaluated < p.evaluated { - return violation("snapshot evaluated count regressed from %d to %d in round %d", p.evaluated, snap.Progress.Evaluated, p.round) - } - return nil -} - -// Run consumes snapshots until a terminal snapshot has been reconciled or the Sweeper -// container has exited. See the package comment for the exit semantics. -func (p *Publisher) Run(ctx context.Context) error { - ticker := time.NewTicker(p.retryDelay()) - defer ticker.Stop() - - for { - // Read the container state BEFORE the snapshot: the Sweeper's last write - // precedes its exit, so an exit observed here means that write is visible below. - state, stateErr := p.Cluster.SweeperState(ctx) - if stateErr != nil { - log.Printf("reading sweeper container state: %v", stateErr) - } - syncErr := p.syncOnce(ctx) - var inputErr *dgdcreconcile.DiffInputError - if errors.As(syncErr, &inputErr) { - return syncErr // malformed projection; retrying cannot fix it - } - if syncErr != nil { - log.Printf("sync: %v", syncErr) - } - - if syncErr == nil && p.terminal { - return p.terminalResult() - } - if stateErr == nil && state.Exited { - return p.finish(ctx, state, syncErr) - } - - select { - case <-ctx.Done(): - return ctx.Err() - case <-ticker.C: - } - } -} - -func (p *Publisher) terminalResult() error { - if p.phase == PhaseFailed { - return ErrRunFailed - } - return nil -} - -// finish handles the Sweeper having exited without a reconciled terminal snapshot. Any -// snapshot it ever wrote is now visible, so the last one is reconciled (with retries, -// since nothing will rewrite it) before deciding the outcome. -func (p *Publisher) finish(ctx context.Context, state SweeperState, syncErr error) error { - for attempt := 1; syncErr != nil; attempt++ { - if attempt >= finalReconcileAttempts { - return fmt.Errorf("reconciling last snapshot: %w", syncErr) - } - select { - case <-ctx.Done(): - return ctx.Err() - case <-time.After(p.retryDelay()): - } - syncErr = p.syncOnce(ctx) - } - switch { - case p.terminal: - return p.terminalResult() - case state.ExitCode != 0: - // Crash or caught failure without a terminal snapshot: the last-known projection - // is reconciled above; the Sweeper's own exit code fails the Job. - return nil - default: - return ErrMissingTerminalSnapshot - } -} - -const finalReconcileAttempts = 3 - -func (p *Publisher) retryDelay() time.Duration { - if p.PollInterval > 0 { - return p.PollInterval - } - return time.Second -} diff --git a/deploy/operator/internal/dgdrrunpublisher/publisher_test.go b/deploy/operator/internal/dgdrrunpublisher/publisher_test.go deleted file mode 100644 index dfd3608f9334..000000000000 --- a/deploy/operator/internal/dgdrrunpublisher/publisher_test.go +++ /dev/null @@ -1,666 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package dgdrrunpublisher - -import ( - "context" - "encoding/json" - "errors" - "os" - "path/filepath" - "reflect" - "strings" - "sync" - "testing" - "time" - - "github.com/ai-dynamo/dynamo/deploy/operator/internal/dgdcreconcile" -) - -// fakeCluster records every mutation so tests can assert exactly which writes happened. -type fakeCluster struct { - mu sync.Mutex - candidates map[string]string // name -> id - created map[string]dgdcreconcile.DesiredCandidate // name -> content at creation - creates []string - deletes []string - patches []RunStatus - ops []string // ordered log of every mutation - incomplete map[string]bool - state SweeperState - patchErr error - createErr error - createCalls int -} - -func newFakeCluster() *fakeCluster { - return &fakeCluster{candidates: map[string]string{}, created: map[string]dgdcreconcile.DesiredCandidate{}, incomplete: map[string]bool{}} -} - -func (f *fakeCluster) ListCandidates(context.Context) ([]dgdcreconcile.CurrentDGDC, error) { - f.mu.Lock() - defer f.mu.Unlock() - var out []dgdcreconcile.CurrentDGDC - for name, id := range f.candidates { - out = append(out, dgdcreconcile.CurrentDGDC{Name: name, ID: id, Incomplete: f.incomplete[name]}) - } - return out, nil -} - -func (f *fakeCluster) CreateCandidate(_ context.Context, name string, c dgdcreconcile.DesiredCandidate) error { - f.mu.Lock() - defer f.mu.Unlock() - f.createCalls++ - if f.createErr != nil { - return f.createErr - } - delete(f.incomplete, name) - f.ops = append(f.ops, "create:"+name) - f.candidates[name] = c.ID - f.created[name] = c - f.creates = append(f.creates, name) - return nil -} - -func (f *fakeCluster) DeleteCandidate(_ context.Context, name string) error { - f.mu.Lock() - defer f.mu.Unlock() - delete(f.candidates, name) - f.ops = append(f.ops, "delete:"+name) - f.deletes = append(f.deletes, name) - return nil -} - -func (f *fakeCluster) PatchRunStatus(_ context.Context, s RunStatus) error { - f.mu.Lock() - defer f.mu.Unlock() - if f.patchErr != nil { - return f.patchErr - } - f.ops = append(f.ops, "patch") - f.patches = append(f.patches, s) - return nil -} - -func (f *fakeCluster) RunUID(context.Context) (string, error) { return testRunUID, nil } - -func (f *fakeCluster) SweeperState(context.Context) (SweeperState, error) { - f.mu.Lock() - defer f.mu.Unlock() - return f.state, nil -} - -func (f *fakeCluster) setState(s SweeperState) { - f.mu.Lock() - defer f.mu.Unlock() - f.state = s -} - -const validManifest = "apiVersion: nvidia.com/v1beta1\nkind: DynamoGraphDeployment\nspec: {}\n" - -const testRunUID = "uid-1" - -func cn(id string) string { return CandidateName("run", testRunUID, id) } - -type cand struct { - id string - failed bool - metrics map[string]any -} - -// snapshotJSON renders a snapshot (JSON is valid YAML, which the real parser accepts). -func snapshotJSON(t *testing.T, phase string, cands ...cand) []byte { - t.Helper() - var list []map[string]any - for _, c := range cands { - entry := map[string]any{"id": c.id, "parameters": map[string]any{"tp": 2}, "metrics": c.metrics} - if c.failed { - entry["outcome"] = OutcomeMaterializationFailed - entry["error"] = "boom" - } else { - entry["outcome"] = OutcomeMaterialized - entry["manifest"] = validManifest - } - list = append(list, entry) - } - data, err := json.Marshal(map[string]any{ - "schemaVersion": 1, - "run": map[string]any{"phase": phase, "terminal": phase != PhaseRunning}, - "progress": map[string]any{"round": 3, "evaluated": len(cands)}, - "candidates": list, - }) - if err != nil { - t.Fatal(err) - } - return data -} - -func writeSnapshot(t *testing.T, dir string, data []byte) { - t.Helper() - if err := os.WriteFile(filepath.Join(dir, SnapshotFileName), data, 0o600); err != nil { - t.Fatal(err) - } -} - -func newPublisher(t *testing.T) (*Publisher, *fakeCluster, string) { - t.Helper() - dir := t.TempDir() - fc := newFakeCluster() - return &Publisher{Cluster: fc, SnapshotDir: dir, RunName: "run", PollInterval: time.Millisecond}, fc, dir -} - -func mustParse(t *testing.T, data []byte) *Snapshot { - t.Helper() - snap, err := ParseSnapshot(data) - if err != nil { - t.Fatal(err) - } - return snap -} - -func TestSwapLeavesBothCandidatesUntouched(t *testing.T) { - p, fc, _ := newPublisher(t) - ctx := context.Background() - if err := p.Reconcile(ctx, mustParse(t, snapshotJSON(t, PhaseRunning, cand{id: "a", metrics: map[string]any{"score": 2.0}}, cand{id: "b", metrics: map[string]any{"score": 1.0}}))); err != nil { - t.Fatal(err) - } - before := map[string]dgdcreconcile.DesiredCandidate{} - for name, content := range fc.created { - before[name] = content - } - if err := p.Reconcile(ctx, mustParse(t, snapshotJSON(t, PhaseRunning, cand{id: "b", metrics: map[string]any{"score": 3.0}}, cand{id: "a", metrics: map[string]any{"score": 2.0}}))); err != nil { - t.Fatal(err) - } - if !reflect.DeepEqual(fc.creates, []string{cn("a"), cn("b")}) || len(fc.deletes) != 0 { - t.Fatalf("swap touched DGDCs: creates=%v deletes=%v", fc.creates, fc.deletes) - } - if !reflect.DeepEqual(before, fc.created) { - t.Fatalf("a published DGDC's content changed: before=%v after=%v", before, fc.created) - } - if got, want := fc.patches[len(fc.patches)-1].CandidateNames, []string{cn("b"), cn("a")}; !reflect.DeepEqual(got, want) { - t.Fatalf("ordered references = %v, want %v", got, want) - } -} - -func TestEvaluatedPointReachesTheClusterComplete(t *testing.T) { - p, fc, _ := newPublisher(t) - if err := p.Reconcile(context.Background(), mustParse(t, snapshotJSON(t, PhaseRunning, cand{id: "a", metrics: map[string]any{"goodputPerGpu": 87.7}}))); err != nil { - t.Fatal(err) - } - got := fc.created[cn("a")] - if got.Spec != validManifest { - t.Fatalf("manifest = %q", got.Spec) - } - if v, ok := got.Parameters["tp"]; !ok || v != float64(2) { - t.Fatalf("resolved evaluation parameters were lost: %v", got.Parameters) - } - if v, ok := got.Metrics["goodputPerGpu"]; !ok || v != 87.7 { - t.Fatalf("evaluation metrics were lost: %v", got.Metrics) - } -} - -func TestReturningCandidateIsRecreatedIdentically(t *testing.T) { - p, fc, _ := newPublisher(t) - ctx := context.Background() - steps := [][]cand{ - {{id: "a"}, {id: "b", metrics: map[string]any{"score": 1.0}}}, - {{id: "a"}}, - {{id: "a"}, {id: "b", metrics: map[string]any{"score": 1.0}}}, - } - var firstB dgdcreconcile.DesiredCandidate - for i, step := range steps { - if err := p.Reconcile(ctx, mustParse(t, snapshotJSON(t, PhaseRunning, step...))); err != nil { - t.Fatal(err) - } - if i == 0 { - firstB = fc.created[cn("b")] - } - } - if !reflect.DeepEqual(fc.creates, []string{cn("a"), cn("b"), cn("b")}) || !reflect.DeepEqual(fc.deletes, []string{cn("b")}) { - t.Fatalf("creates=%v deletes=%v", fc.creates, fc.deletes) - } - if !reflect.DeepEqual(firstB, fc.created[cn("b")]) { - t.Fatalf("recreated candidate differs: %v vs %v", firstB, fc.created[cn("b")]) - } -} - -func TestRepeatedSnapshotCausesNoWrites(t *testing.T) { - p, fc, _ := newPublisher(t) - ctx := context.Background() - snap := mustParse(t, snapshotJSON(t, PhaseRunning, cand{id: "a"})) - for i := 0; i < 3; i++ { - if err := p.Reconcile(ctx, snap); err != nil { - t.Fatal(err) - } - } - if len(fc.creates) != 1 || len(fc.patches) != 1 || len(fc.deletes) != 0 { - t.Fatalf("creates=%v patches=%d deletes=%v", fc.creates, len(fc.patches), fc.deletes) - } -} - -func TestDroppedCandidateIsDeletedAfterStatusPatch(t *testing.T) { - p, fc, _ := newPublisher(t) - ctx := context.Background() - _ = p.Reconcile(ctx, mustParse(t, snapshotJSON(t, PhaseRunning, cand{id: "a"}, cand{id: "b"}))) - _ = p.Reconcile(ctx, mustParse(t, snapshotJSON(t, PhaseRunning, cand{id: "a"}))) - if !reflect.DeepEqual(fc.deletes, []string{cn("b")}) { - t.Fatalf("deletes = %v", fc.deletes) - } - if got := fc.patches[len(fc.patches)-1].CandidateNames; !reflect.DeepEqual(got, []string{cn("a")}) { - t.Fatalf("status refs = %v", got) - } - want := []string{"create:" + cn("a"), "create:" + cn("b"), "patch", "patch", "delete:" + cn("b")} - if !reflect.DeepEqual(fc.ops, want) { - t.Fatalf("ops = %v, want %v", fc.ops, want) - } -} - -func TestFailedMaterializationIsNotCreatedButCounted(t *testing.T) { - p, fc, _ := newPublisher(t) - if err := p.Reconcile(context.Background(), mustParse(t, snapshotJSON(t, PhaseRunning, cand{id: "a"}, cand{id: "x", failed: true}))); err != nil { - t.Fatal(err) - } - if !reflect.DeepEqual(fc.creates, []string{cn("a")}) { - t.Fatalf("creates = %v", fc.creates) - } - if got := fc.patches[0].CandidateNames; !reflect.DeepEqual(got, []string{cn("a")}) { - t.Fatalf("refs must not include the failed candidate: %v", got) - } - if msg := fc.patches[0].Message; !strings.Contains(msg, "1 candidate(s) failed materialization") { - t.Fatalf("message = %q", msg) - } -} - -func withTimestamp(t *testing.T, data []byte, timestamp string) []byte { - t.Helper() - var doc map[string]any - if err := json.Unmarshal(data, &doc); err != nil { - t.Fatal(err) - } - doc["timestamp"] = timestamp - out, err := json.Marshal(doc) - if err != nil { - t.Fatal(err) - } - return out -} - -func TestLastProgressComesFromTheSnapshotTimestamp(t *testing.T) { - p, fc, _ := newPublisher(t) - data := withTimestamp(t, snapshotJSON(t, PhaseRunning, cand{id: "a"}), "2026-10-05T21:35:12Z") - if err := p.Reconcile(context.Background(), mustParse(t, data)); err != nil { - t.Fatal(err) - } - want := time.Date(2026, 10, 5, 21, 35, 12, 0, time.UTC) - if got := fc.patches[0].LastProgress; !got.Equal(want) { - t.Fatalf("LastProgress = %v, want %v", got, want) - } -} - -func TestProgressWithoutRankChangeStillPatchesTheProgressTime(t *testing.T) { - p, fc, _ := newPublisher(t) - first := withTimestamp(t, snapshotJSON(t, PhaseRunning, cand{id: "a"}), "2026-10-05T21:35:12Z") - second := withTimestamp(t, snapshotJSON(t, PhaseRunning, cand{id: "a"}), "2026-10-05T21:35:14Z") - for _, data := range [][]byte{first, second, second} { - if err := p.Reconcile(context.Background(), mustParse(t, data)); err != nil { - t.Fatal(err) - } - } - if len(fc.patches) != 2 { - t.Fatalf("want one patch per distinct progress time, got %d", len(fc.patches)) - } - if !reflect.DeepEqual(fc.patches[0].CandidateNames, fc.patches[1].CandidateNames) { - t.Fatalf("refs changed: %v", fc.patches) - } - if len(fc.creates) != 1 { - t.Fatalf("an unchanged candidate must not be created again: %v", fc.creates) - } -} - -func TestMissingOrMalformedTimestampIsNotAnError(t *testing.T) { - for _, timestamp := range []string{"", "yesterday"} { - p, fc, _ := newPublisher(t) - data := withTimestamp(t, snapshotJSON(t, PhaseRunning, cand{id: "a"}), timestamp) - if err := p.Reconcile(context.Background(), mustParse(t, data)); err != nil { - t.Fatalf("timestamp %q: %v", timestamp, err) - } - if got := fc.patches[0].LastProgress; !got.IsZero() { - t.Fatalf("timestamp %q: LastProgress = %v, want zero", timestamp, got) - } - } -} - -func TestCreateFailureDoesNotPatchStatus(t *testing.T) { - p, fc, _ := newPublisher(t) - fc.createErr = errors.New("apiserver down") - if err := p.Reconcile(context.Background(), mustParse(t, snapshotJSON(t, PhaseRunning, cand{id: "a"}))); err == nil { - t.Fatal("want error") - } - if len(fc.patches) != 0 { - t.Fatalf("status must not reference candidates that were not created: %v", fc.patches) - } -} - -func TestParseSnapshotValidation(t *testing.T) { - bad := map[string]string{ - "schema": `{"schemaVersion":2,"run":{"phase":"Running","terminal":false}}`, - "phase": `{"schemaVersion":1,"run":{"phase":"Weird","terminal":false}}`, - "terminal flag": `{"schemaVersion":1,"run":{"phase":"Succeeded","terminal":false}}`, - "duplicate": `{"schemaVersion":1,"run":{"phase":"Running","terminal":false},"candidates":[{"id":"a","outcome":"materialized","manifest":"apiVersion: nvidia.com/v1beta1\\nkind: DynamoGraphDeployment\\nspec: {}\\n"},{"id":"a","outcome":"materialized","manifest":"apiVersion: nvidia.com/v1beta1\\nkind: DynamoGraphDeployment\\nspec: {}\\n"}]}`, - "no manifest": `{"schemaVersion":1,"run":{"phase":"Running","terminal":false},"candidates":[{"id":"a","outcome":"materialized"}]}`, - "no error": `{"schemaVersion":1,"run":{"phase":"Running","terminal":false},"candidates":[{"id":"a","outcome":"materialization_failed"}]}`, - "unsafe id": `{"schemaVersion":1,"run":{"phase":"Running","terminal":false},"candidates":[{"id":"Not_Valid","outcome":"materialized","manifest":"apiVersion: nvidia.com/v1beta1\\nkind: DynamoGraphDeployment\\nspec: {}\\n"}]}`, - "long id": `{"schemaVersion":1,"run":{"phase":"Running","terminal":false},"candidates":[{"id":"%s","outcome":"materialized","manifest":"apiVersion: nvidia.com/v1beta1\\nkind: DynamoGraphDeployment\\nspec: {}\\n"}]}`, - "wrong kind": `{"schemaVersion":1,"run":{"phase":"Running","terminal":false},"candidates":[{"id":"a","outcome":"materialized","manifest":"kind: ConfigMap\\nspec: {}\\n"}]}`, - "empty id": `{"schemaVersion":1,"run":{"phase":"Running","terminal":false},"candidates":[{"id":"","outcome":"materialized","manifest":"apiVersion: nvidia.com/v1beta1\\nkind: DynamoGraphDeployment\\nspec: {}\\n"}]}`, - } - bad["long id"] = strings.Replace(bad["long id"], "%s", strings.Repeat("a", 64), 1) - for name, doc := range bad { - if _, err := ParseSnapshot([]byte(doc)); !errors.Is(err, ErrProtocolViolation) { - t.Errorf("%s: want protocol violation, got %v", name, err) - } - } -} - -func runWithTimeout(t *testing.T, p *Publisher) error { - t.Helper() - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - return p.Run(ctx) -} - -func lastPatch(t *testing.T, fc *fakeCluster) RunStatus { - t.Helper() - if len(fc.patches) == 0 { - t.Fatal("no run status patch was written") - } - return fc.patches[len(fc.patches)-1] -} - -// Orderly completion: terminal snapshot is reconciled and the publisher exits 0 even -// though the Sweeper container is still shutting down. -func TestRunTerminalSnapshotAcknowledges(t *testing.T) { - p, fc, dir := newPublisher(t) - writeSnapshot(t, dir, snapshotJSON(t, PhaseSucceeded, cand{id: "a"})) - err := runWithTimeout(t, p) - if err != nil || ExitCode(err) != ExitAcknowledged { - t.Fatalf("err = %v", err) - } - if got := lastPatch(t, fc).CandidateNames; !reflect.DeepEqual(got, []string{cn("a")}) { - t.Fatalf("refs = %v", got) - } -} - -func TestRunTerminalFailedSnapshotIsReconciledThenFails(t *testing.T) { - p, fc, dir := newPublisher(t) - writeSnapshot(t, dir, snapshotJSON(t, PhaseFailed, cand{id: "a"})) - err := runWithTimeout(t, p) - if !errors.Is(err, ErrRunFailed) || ExitCode(err) != ExitRunFailed { - t.Fatalf("err = %v", err) - } - if !reflect.DeepEqual(fc.creates, []string{cn("a")}) { - t.Fatalf("creates = %v", fc.creates) - } -} - -func TestRunReconcilesTerminalSnapshotWrittenAfterRunningOne(t *testing.T) { - p, fc, dir := newPublisher(t) - writeSnapshot(t, dir, snapshotJSON(t, PhaseRunning, cand{id: "a"})) - done := make(chan error, 1) - go func() { done <- runWithTimeout(t, p) }() - time.Sleep(20 * time.Millisecond) - writeSnapshot(t, dir, snapshotJSON(t, PhaseSucceeded, cand{id: "a"}, cand{id: "b"})) - if err := <-done; err != nil { - t.Fatal(err) - } - if got := lastPatch(t, fc).CandidateNames; len(got) != 2 { - t.Fatalf("terminal projection lost: %v", got) - } -} - -// Abrupt non-zero exit without a terminal snapshot: the last snapshot is reconciled and -// the publisher exits 0. -func TestRunCrashWithoutTerminalSnapshotReconcilesLastSnapshot(t *testing.T) { - p, fc, dir := newPublisher(t) - writeSnapshot(t, dir, snapshotJSON(t, PhaseRunning, cand{id: "a"})) - fc.setState(SweeperState{Exited: true, ExitCode: 137}) - if err := runWithTimeout(t, p); err != nil { - t.Fatal(err) - } - if got := lastPatch(t, fc).CandidateNames; !reflect.DeepEqual(got, []string{cn("a")}) { - t.Fatalf("refs = %v", got) - } -} - -// Nothing was ever written and the Sweeper crashed: nothing to reconcile, exit 0. -func TestRunCrashBeforeAnySnapshot(t *testing.T) { - p, fc, _ := newPublisher(t) - fc.setState(SweeperState{Exited: true, ExitCode: 137}) - if err := runWithTimeout(t, p); err != nil { - t.Fatal(err) - } - if len(fc.creates) != 0 || len(fc.patches) != 0 { - t.Fatalf("unexpected writes: %v %v", fc.creates, fc.patches) - } -} - -// Exit 0 without a terminal snapshot is a protocol violation: the last snapshot is still -// reconciled but the publisher fails with the dedicated exit code. -func TestRunCleanExitWithoutTerminalSnapshotIsProtocolViolation(t *testing.T) { - p, fc, dir := newPublisher(t) - writeSnapshot(t, dir, snapshotJSON(t, PhaseRunning, cand{id: "a"})) - fc.setState(SweeperState{Exited: true, ExitCode: 0}) - err := runWithTimeout(t, p) - if !errors.Is(err, ErrMissingTerminalSnapshot) || ExitCode(err) != ExitProtocolViolation { - t.Fatalf("err = %v", err) - } - if !reflect.DeepEqual(fc.creates, []string{cn("a")}) { - t.Fatalf("last snapshot must still be reconciled: %v", fc.creates) - } -} - -func TestRunCleanExitWithNoSnapshotAtAllIsProtocolViolation(t *testing.T) { - p, fc, _ := newPublisher(t) - fc.setState(SweeperState{Exited: true, ExitCode: 0}) - if err := runWithTimeout(t, p); !errors.Is(err, ErrMissingTerminalSnapshot) { - t.Fatalf("err = %v", err) - } -} - -func TestRunFinalReconcileFailureFails(t *testing.T) { - p, fc, dir := newPublisher(t) - writeSnapshot(t, dir, snapshotJSON(t, PhaseRunning, cand{id: "a"})) - fc.createErr = errors.New("apiserver down") - fc.setState(SweeperState{Exited: true, ExitCode: 1}) - err := runWithTimeout(t, p) - if err == nil || ExitCode(err) != ExitReconcileFailed { - t.Fatalf("err = %v", err) - } - if fc.createCalls < finalReconcileAttempts { - t.Fatalf("expected %d attempts, got %d", finalReconcileAttempts, fc.createCalls) - } -} - -func TestRunTerminalSnapshotReconcileFailureIsRetriedUntilSuccess(t *testing.T) { - p, fc, dir := newPublisher(t) - writeSnapshot(t, dir, snapshotJSON(t, PhaseSucceeded, cand{id: "a"})) - fc.createErr = errors.New("flaky") - done := make(chan error, 1) - go func() { done <- runWithTimeout(t, p) }() - time.Sleep(20 * time.Millisecond) - fc.mu.Lock() - fc.createErr = nil - fc.mu.Unlock() - if err := <-done; err != nil { - t.Fatal(err) - } - if !reflect.DeepEqual(fc.creates, []string{cn("a")}) { - t.Fatalf("creates = %v", fc.creates) - } -} - -func TestRunMalformedProjectionFailsImmediately(t *testing.T) { - p, fc, dir := newPublisher(t) - fc.candidates["x"] = "dup" - fc.candidates["y"] = "dup" - writeSnapshot(t, dir, snapshotJSON(t, PhaseRunning, cand{id: "a"})) - err := runWithTimeout(t, p) - var inputErr *dgdcreconcile.DiffInputError - if !errors.As(err, &inputErr) { - t.Fatalf("err = %v", err) - } -} - -func TestRunContextCancelledReturnsError(t *testing.T) { - p, _, _ := newPublisher(t) - ctx, cancel := context.WithCancel(context.Background()) - cancel() - if err := p.Run(ctx); !errors.Is(err, context.Canceled) { - t.Fatalf("err = %v", err) - } -} - -func TestCandidateNameIsBoundedAndUniquePerRunIncarnation(t *testing.T) { - id := "evaluated-point-0123456789ab" - if cn("a") != CandidateName("run", testRunUID, "a") || cn("a") == CandidateName("run", "uid-2", "a") { - t.Fatalf("a recreated run must get different names: %q", cn("a")) - } - long := strings.Repeat("r", 253) - other := strings.Repeat("r", 252) + "s" - name := CandidateName(long, testRunUID, id) - if len(name) > 253 || !strings.HasSuffix(name, "-"+id) || name == CandidateName(other, testRunUID, id) || name != CandidateName(long, testRunUID, id) { - t.Fatalf("name = %q (%d)", name, len(name)) - } -} - -func TestUnchangedSnapshotRepairsDrift(t *testing.T) { - p, fc, dir := newPublisher(t) - ctx := context.Background() - writeSnapshot(t, dir, snapshotJSON(t, PhaseRunning, cand{id: "a"})) - if err := p.syncOnce(ctx); err != nil { - t.Fatal(err) - } - delete(fc.candidates, cn("a")) // someone deletes the referenced candidate - if err := p.syncOnce(ctx); err != nil { - t.Fatal(err) - } - if !reflect.DeepEqual(fc.creates, []string{cn("a"), cn("a")}) { - t.Fatalf("creates = %v", fc.creates) - } -} - -func TestCandidateWithoutStatusIsCompletedBeforeItIsReferenced(t *testing.T) { - p, fc, dir := newPublisher(t) - ctx := context.Background() - fc.candidates[cn("a")] = "a" - fc.incomplete[cn("a")] = true // created, but the status write failed - writeSnapshot(t, dir, snapshotJSON(t, PhaseRunning, cand{id: "a"})) - if err := p.syncOnce(ctx); err != nil { - t.Fatal(err) - } - if want := []string{"create:" + cn("a"), "patch"}; !reflect.DeepEqual(fc.ops, want) { - t.Fatalf("ops = %v, want %v", fc.ops, want) - } -} - -func writeProgress(t *testing.T, dir, phase string, round, evaluated int) { - t.Helper() - doc := map[string]any{} - if err := json.Unmarshal(snapshotJSON(t, phase, cand{id: "a"}), &doc); err != nil { - t.Fatal(err) - } - doc["progress"] = map[string]any{"round": round, "evaluated": evaluated} - data, err := json.Marshal(doc) - if err != nil { - t.Fatal(err) - } - writeSnapshot(t, dir, data) -} - -func TestRegressingSnapshotIsRejectedBeforeReconcile(t *testing.T) { - for name, regressed := range map[string][2]int{"older round": {2, 99}, "fewer evaluated in the same round": {3, 4}} { - t.Run(name, func(t *testing.T) { - p, fc, dir := newPublisher(t) - ctx := context.Background() - writeProgress(t, dir, PhaseRunning, 3, 5) - if err := p.syncOnce(ctx); err != nil { - t.Fatal(err) - } - ops := len(fc.ops) - writeProgress(t, dir, PhaseRunning, regressed[0], regressed[1]) - err := p.syncOnce(ctx) - if !errors.Is(err, ErrProtocolViolation) || ExitCode(err) != ExitProtocolViolation { - t.Fatalf("err = %v", err) - } - if len(fc.ops) != ops { - t.Fatalf("a regressing snapshot must not be reconciled: %v", fc.ops[ops:]) - } - }) - } -} - -func TestTerminalSnapshotMayReportItsFinalEvaluatedTotal(t *testing.T) { - p, _, dir := newPublisher(t) - ctx := context.Background() - writeProgress(t, dir, PhaseRunning, 3, 9) - if err := p.syncOnce(ctx); err != nil { - t.Fatal(err) - } - writeProgress(t, dir, PhaseSucceeded, 3, 7) - if err := p.syncOnce(ctx); err != nil { - t.Fatalf("terminal snapshot rejected: %v", err) - } -} - -func TestInvalidFinalSnapshotIsAProtocolViolationExit(t *testing.T) { - p, fc, dir := newPublisher(t) - writeSnapshot(t, dir, []byte(`{"schemaVersion":2}`)) - fc.setState(SweeperState{Exited: true, ExitCode: 0}) - err := runWithTimeout(t, p) - if ExitCode(err) != ExitProtocolViolation { - t.Fatalf("err = %v, exit = %d", err, ExitCode(err)) - } -} - -func TestParseManifestHandlesCompanionResources(t *testing.T) { - dgd := "apiVersion: nvidia.com/v1beta1\nkind: DynamoGraphDeployment\nspec: {}\n" - cm := "apiVersion: v1\nkind: ConfigMap\nmetadata: {}\n" - for name, manifest := range map[string]string{ - "dgd first": dgd + "---\n" + cm, - "configmap first": "# generated\n---\n" + cm + "---\n" + dgd, - } { - got, err := ParseManifest(manifest) - if err != nil { - t.Fatalf("%s: %v", name, err) - } - if got.DGD["kind"] != "DynamoGraphDeployment" || len(got.Companions) != 1 || !strings.Contains(got.Companions[0], "ConfigMap") { - t.Fatalf("%s: %+v", name, got) - } - } - for name, manifest := range map[string]string{ - "two dgds": dgd + "---\n" + dgd, - "no dgd": cm, - "no spec": "apiVersion: nvidia.com/v1beta1\nkind: DynamoGraphDeployment\n", - "no apiVersion": "kind: DynamoGraphDeployment\nspec: {}\n", - "unsupported apiVersion": "apiVersion: nvidia.com/v9\nkind: DynamoGraphDeployment\nspec: {}\n", - } { - if _, err := ParseManifest(manifest); err == nil { - t.Errorf("%s: want error", name) - } - } -} diff --git a/deploy/operator/internal/dgdrrunpublisher/reconciler.go b/deploy/operator/internal/dgdrrunpublisher/reconciler.go new file mode 100644 index 000000000000..184a91d3492b --- /dev/null +++ b/deploy/operator/internal/dgdrrunpublisher/reconciler.go @@ -0,0 +1,402 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dgdrrunpublisher + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "log" + "strings" + "time" + + dgdv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1" + v1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/dgdcreconcile" + v1beta2 "github.com/ai-dynamo/dynamo/deploy/operator/internal/dgdrrunpublisher/placeholderapi" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" +) + +const ( + // LabelRunUID ties a candidate to one incarnation of its run. The run name can be + // reused after a delete and recreate; the UID cannot, so candidates left over from a + // previous run of the same name are never mistaken for current ones. The manager's + // cache selects candidates by this label. + LabelRunUID = "nvidia.com/dgdr-run-uid" + // LabelCandidateID records the stable candidate id (the identity read back on list). + LabelCandidateID = "nvidia.com/dgdr-candidate-id" + + evaluatedCondition = "Evaluated" + + // AnnotationAdditionalResources carries the companion resources (for example + // generated ConfigMaps) rendered next to the DGD, as multi-document YAML. It is the + // same key and format the DGDR controller uses. + AnnotationAdditionalResources = "dgdr.nvidia.com/additional-resources" + + // fieldOwner is the server-side-apply field manager of the publisher. + fieldOwner = "dgdr-run-publisher" + + // finalReconcileAttempts bounds the retries once the Sweeper has exited: nothing will + // rewrite the snapshot, so a persistent failure must end the process. + finalReconcileAttempts = 3 +) + +// Reconciler reconciles the one DynamoGraphDeploymentRun this publisher serves. +// +// Client is the manager's client: reads come from the scoped informer cache (this Run, +// its candidates, this Pod), writes go to the API server. It needs RBAC (namespaced Role +// bound to the run's ServiceAccount): runs get/list/watch and runs/status patch (server- +// side apply); candidates get/list/watch/create/patch/delete and candidates/status patch; +// pods get/list/watch (own pod, for Sweeper crash detection; resourceNames cannot be used +// with list/watch of generated Job pod names, so the cache narrows with a field selector). +type Reconciler struct { + Client client.Client + Namespace string + RunName string + RunUID string + PodName string + SweeperContainer string + + Observer *Observer + Store *InputStore + Completion *Completion + + // finalFailures counts failed reconciles after the Sweeper exited. The controller + // runs one worker, so it needs no locking. + finalFailures int +} + +func (r *Reconciler) key() types.NamespacedName { + return types.NamespacedName{Namespace: r.Namespace, Name: r.RunName} +} + +// Reconcile handles the single run key. Both the input observer and informer events +// enqueue it; the returned error makes the workqueue retry with backoff. +func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + if req.NamespacedName != r.key() || r.Completion.Finished() { + return ctrl.Result{}, nil + } + return ctrl.Result{}, r.reconcile(ctx) +} + +func (r *Reconciler) reconcile(ctx context.Context) error { + var run v1beta2.DynamoGraphDeploymentRun + if err := r.Client.Get(ctx, r.key(), &run); err != nil { + return fmt.Errorf("getting run: %w", err) + } + if string(run.UID) != r.RunUID { + // The run was deleted and recreated under the same name; this pod belongs to the + // previous incarnation and must not touch the new one. + r.Completion.Finish(fmt.Errorf("run %s has uid %s, this publisher serves uid %s", r.RunName, run.UID, r.RunUID)) + return nil + } + + // Read the container state BEFORE the snapshot: the Sweeper's last write precedes its + // exit, so an exit observed here means that write is visible to the check below. + state, stateErr := r.sweeperState(ctx) + if stateErr != nil { + log.Printf("reading sweeper container state: %v", stateErr) + } + exited := stateErr == nil && state.Exited + if exited { + r.Observer.Check() + } + + input := r.Store.Load() + if input.Err != nil && exited { + if errors.Is(input.Err, ErrProtocolViolation) { + r.Completion.Finish(input.Err) + return nil + } + return r.retryOrFail(exited, input.Err) + } + if input.Snapshot == nil { + if exited { + r.Completion.Finish(exitResult(state)) + } + return nil // the observer enqueues the key when the first snapshot arrives + } + + desired := Desire(r.RunName, r.RunUID, input.Snapshot) + if err := r.apply(ctx, &run, desired); err != nil { + return r.retryOrFail(exited, err) + } + switch { + case desired.Terminal: + r.Completion.Finish(terminalResult(desired.Phase)) + case exited: + r.Completion.Finish(exitResult(state)) + } + return nil +} + +// retryOrFail hands a failure to the workqueue's backoff, except where retrying cannot +// help (malformed projection) or the Sweeper is gone and the attempts are used up. +func (r *Reconciler) retryOrFail(exited bool, err error) error { + var inputErr *dgdcreconcile.DiffInputError + if errors.As(err, &inputErr) { + r.Completion.Finish(err) // retrying cannot fix it + return nil + } + if exited { + r.finalFailures++ + if r.finalFailures >= finalReconcileAttempts { + r.Completion.Finish(fmt.Errorf("reconciling last snapshot: %w", err)) + return nil + } + } + return err +} + +// apply drives the cluster toward the desired state: create missing candidates, finish +// the status of candidates that lack it, update the run, then delete obsolete candidates. +// It is idempotent and safe to retry; existing candidate specs are never touched. +func (r *Reconciler) apply(ctx context.Context, run *v1beta2.DynamoGraphDeploymentRun, desired DesiredState) error { + var list v1beta2.DynamoGraphDeploymentCandidateList + if err := r.Client.List(ctx, &list, client.InNamespace(r.Namespace), client.MatchingLabels{LabelRunUID: r.RunUID}); err != nil { + return fmt.Errorf("listing candidates: %w", err) + } + current := make([]dgdcreconcile.CurrentDGDC, 0, len(list.Items)) + uids := make(map[string]types.UID, len(list.Items)) + for i := range list.Items { + item := &list.Items[i] + current = append(current, dgdcreconcile.CurrentDGDC{ + Name: item.Name, + ID: item.Labels[LabelCandidateID], + Incomplete: !meta.IsStatusConditionTrue(item.Status.Conditions, evaluatedCondition), + }) + uids[item.Name] = item.UID + } + actions, err := dgdcreconcile.ComputeActions(desired.Candidates, current) + if err != nil { + return err + } + + for _, candidate := range actions.Creates { + if err := r.createCandidate(ctx, run, desired.Names[candidate.ID], candidate); err != nil { + return fmt.Errorf("creating candidate %s: %w", candidate.ID, err) + } + } + unpopulated := make([]dgdcreconcile.DesiredCandidate, 0, len(actions.Creates)+len(actions.Incomplete)) + unpopulated = append(unpopulated, actions.Creates...) + unpopulated = append(unpopulated, actions.Incomplete...) + for _, candidate := range unpopulated { + if err := r.populateStatus(ctx, desired.Names[candidate.ID], candidate); err != nil { + return fmt.Errorf("populating status of candidate %s: %w", candidate.ID, err) + } + } + + if desired.Status.NeedsUpdate(observedStatus(run)) { + if err := r.updateRunStatus(ctx, desired.Status); err != nil { + return fmt.Errorf("updating run status: %w", err) + } + } + + for _, name := range actions.Deletes { + uid := uids[name] + object := &v1beta2.DynamoGraphDeploymentCandidate{ObjectMeta: metav1.ObjectMeta{Namespace: r.Namespace, Name: name}} + if err := client.IgnoreNotFound(r.Client.Delete(ctx, object, client.Preconditions{UID: &uid})); err != nil { + return fmt.Errorf("deleting candidate %s: %w", name, err) + } + } + return nil +} + +// createCandidate creates the candidate with server-side apply. It is only called for a +// candidate that is missing from the cache, and re-applying the same content after a +// stale cache read is a no-op, so no spec comparison is needed. +func (r *Reconciler) createCandidate(ctx context.Context, run *v1beta2.DynamoGraphDeploymentRun, name string, candidate dgdcreconcile.DesiredCandidate) error { + spec, companions, err := decodeManifest(candidate) + if err != nil { + return err + } + parameters, err := normalizeJSON(candidate.Parameters) + if err != nil { + return err + } + object := &v1beta2.DynamoGraphDeploymentCandidate{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: r.Namespace, + Name: name, + Labels: map[string]string{LabelRunUID: r.RunUID, LabelCandidateID: candidate.ID}, + }, + Spec: v1beta2.DynamoGraphDeploymentCandidateSpec{DynamoGraphDeploymentSpec: *spec}, + } + if companions != "" { + object.Annotations = map[string]string{AnnotationAdditionalResources: companions} + } + if err := controllerutil.SetControllerReference(run, object, r.Client.Scheme()); err != nil { + return err + } + content, err := runtime.DefaultUnstructuredConverter.ToUnstructured(object) + if err != nil { + return err + } + // The status is written separately, through the status subresource. + delete(content, "status") + unstructured.RemoveNestedField(content, "metadata", "creationTimestamp") + if err := unstructured.SetNestedField(content, parameters, "spec", "parameters"); err != nil { + return err + } + apply := &unstructured.Unstructured{Object: content} + apply.SetGroupVersionKind(v1beta2.GroupVersion.WithKind("DynamoGraphDeploymentCandidate")) + return r.Client.Apply(ctx, client.ApplyConfigurationFromUnstructured(apply), client.FieldOwner(fieldOwner), client.ForceOwnership) +} + +// populateStatus records the evaluation facts once. The candidate is referenced from the +// run only after this, and its status is immutable afterwards. +func (r *Reconciler) populateStatus(ctx context.Context, name string, candidate dgdcreconcile.DesiredCandidate) error { + metrics, err := normalizeJSON(candidate.Metrics) + if err != nil { + return err + } + apply := &unstructured.Unstructured{Object: map[string]any{ + "metadata": map[string]any{"namespace": r.Namespace, "name": name}, + "status": map[string]any{ + "metrics": metrics, + "conditions": []any{map[string]any{ + "type": evaluatedCondition, + "status": string(metav1.ConditionTrue), + "reason": evaluatedCondition, + "message": "evaluated by the Sweeper", + "lastTransitionTime": time.Now().UTC().Format(time.RFC3339), + }}, + }, + }} + apply.SetGroupVersionKind(v1beta2.GroupVersion.WithKind("DynamoGraphDeploymentCandidate")) + return r.Client.Status().Apply(ctx, client.ApplyConfigurationFromUnstructured(apply), client.FieldOwner(fieldOwner), client.ForceOwnership) +} + +// updateRunStatus writes the publisher-owned run status fields with server-side apply, so +// the run controller's own status fields (conditions) are left alone. +func (r *Reconciler) updateRunStatus(ctx context.Context, status RunStatus) error { + refs := make([]any, 0, len(status.CandidateNames)) + for _, name := range status.CandidateNames { + refs = append(refs, map[string]any{"name": name}) + } + fields := map[string]any{ + "message": status.Message, + "candidateRefs": refs, + } + if status.HasProgress { + fields["progress"] = map[string]any{"round": int64(status.Round), "evaluated": int64(status.Evaluated)} + } + if !status.LastProgress.IsZero() { + fields["lastProgressTime"] = status.LastProgress.UTC().Format(time.RFC3339) + } + apply := &unstructured.Unstructured{Object: map[string]any{ + "metadata": map[string]any{"namespace": r.Namespace, "name": r.RunName}, + "status": fields, + }} + apply.SetGroupVersionKind(v1beta2.GroupVersion.WithKind("DynamoGraphDeploymentRun")) + return r.Client.Status().Apply(ctx, client.ApplyConfigurationFromUnstructured(apply), client.FieldOwner(fieldOwner), client.ForceOwnership) +} + +// observedStatus is the publisher-owned part of the run's current (cached) status. +func observedStatus(run *v1beta2.DynamoGraphDeploymentRun) RunStatus { + observed := RunStatus{Message: run.Status.Message} + if progress := run.Status.Progress; progress != nil { + observed.HasProgress = true + observed.Round = progress.Round + observed.Evaluated = progress.Evaluated + } + if run.Status.LastProgressTime != nil { + observed.LastProgress = run.Status.LastProgressTime.UTC() + } + for _, ref := range run.Status.CandidateRefs { + observed.CandidateNames = append(observed.CandidateNames, ref.Name) + } + return observed +} + +// sweeperState reads the Sweeper container's status from this pod, via the cache. +func (r *Reconciler) sweeperState(ctx context.Context) (SweeperState, error) { + var pod corev1.Pod + if err := r.Client.Get(ctx, types.NamespacedName{Namespace: r.Namespace, Name: r.PodName}, &pod); err != nil { + return SweeperState{}, err + } + return sweeperStateOf(&pod, r.SweeperContainer), nil +} + +func sweeperStateOf(pod *corev1.Pod, container string) SweeperState { + for _, status := range pod.Status.ContainerStatuses { + if status.Name != container { + continue + } + if terminated := status.State.Terminated; terminated != nil { + return SweeperState{Exited: true, ExitCode: terminated.ExitCode} + } + return SweeperState{} + } + return SweeperState{} +} + +// normalizeJSON round-trips a value through JSON so it only holds the types unstructured +// objects allow (int64, float64, string, bool, maps, slices). +func normalizeJSON(value map[string]any) (map[string]any, error) { + out := map[string]any{} + if value == nil { + return out, nil + } + raw, err := json.Marshal(value) + if err != nil { + return nil, err + } + if err := json.Unmarshal(raw, &out); err != nil { + return nil, err + } + return out, nil +} + +// decodeManifest extracts the DGD spec and the companion resources from a rendered +// manifest. A DGD in either supported apiVersion is converted to v1beta1, the same way +// the DGDR controller does it. +func decodeManifest(candidate dgdcreconcile.DesiredCandidate) (*v1beta1.DynamoGraphDeploymentSpec, string, error) { + manifest, err := ParseManifest(candidate.Spec) + if err != nil { + return nil, "", fmt.Errorf("candidate %s: %w", candidate.ID, err) + } + obj := &unstructured.Unstructured{Object: manifest.DGD} + var beta v1beta1.DynamoGraphDeployment + switch obj.GetAPIVersion() { + case dgdv1alpha1.GroupVersion.String(): + var alpha dgdv1alpha1.DynamoGraphDeployment + if err := runtime.DefaultUnstructuredConverter.FromUnstructured(obj.Object, &alpha); err != nil { + return nil, "", fmt.Errorf("candidate %s: %w", candidate.ID, err) + } + if err := alpha.ConvertTo(&beta); err != nil { + return nil, "", fmt.Errorf("candidate %s: %w", candidate.ID, err) + } + case v1beta1.GroupVersion.String(): + if err := runtime.DefaultUnstructuredConverter.FromUnstructured(obj.Object, &beta); err != nil { + return nil, "", fmt.Errorf("candidate %s: %w", candidate.ID, err) + } + default: + return nil, "", fmt.Errorf("candidate %s: unsupported DynamoGraphDeployment apiVersion %q", candidate.ID, obj.GetAPIVersion()) + } + return &beta.Spec, strings.Join(manifest.Companions, "\n---\n"), nil +} diff --git a/deploy/operator/internal/dgdrrunpublisher/reconciler_test.go b/deploy/operator/internal/dgdrrunpublisher/reconciler_test.go new file mode 100644 index 000000000000..52467339ab07 --- /dev/null +++ b/deploy/operator/internal/dgdrrunpublisher/reconciler_test.go @@ -0,0 +1,101 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dgdrrunpublisher + +import ( + "reflect" + "testing" + "time" + + v1beta2 "github.com/ai-dynamo/dynamo/deploy/operator/internal/dgdrrunpublisher/placeholderapi" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func podWith(statuses ...corev1.ContainerStatus) *corev1.Pod { + return &corev1.Pod{Status: corev1.PodStatus{ContainerStatuses: statuses}} +} + +func TestSweeperStateOfReadsOnlyTheSweeperContainer(t *testing.T) { + terminated := func(name string, code int32) corev1.ContainerStatus { + return corev1.ContainerStatus{Name: name, State: corev1.ContainerState{Terminated: &corev1.ContainerStateTerminated{ExitCode: code}}} + } + running := corev1.ContainerStatus{Name: "sweeper", State: corev1.ContainerState{Running: &corev1.ContainerStateRunning{}}} + + cases := map[string]struct { + pod *corev1.Pod + want SweeperState + }{ + "no statuses yet": {podWith(), SweeperState{}}, + "running": {podWith(running), SweeperState{}}, + "exited non-zero": {podWith(terminated("sweeper", 3)), SweeperState{Exited: true, ExitCode: 3}}, + "exited zero": {podWith(terminated("sweeper", 0)), SweeperState{Exited: true}}, + "other container only": {podWith(terminated("publisher", 1)), SweeperState{}}, + } + for name, tc := range cases { + if got := sweeperStateOf(tc.pod, "sweeper"); got != tc.want { + t.Errorf("%s: got %+v, want %+v", name, got, tc.want) + } + } +} + +func TestObservedStatusMirrorsTheRunStatus(t *testing.T) { + when := metav1.NewTime(time.Date(2026, 10, 5, 21, 35, 12, 0, time.UTC)) + run := &v1beta2.DynamoGraphDeploymentRun{Status: v1beta2.DynamoGraphDeploymentRunStatus{ + Message: "m", + Progress: &v1beta2.RunProgress{Round: 2, Evaluated: 5}, + LastProgressTime: &when, + CandidateRefs: []v1beta2.CandidateRef{{Name: "x"}, {Name: "y"}}, + }} + want := RunStatus{Message: "m", HasProgress: true, Round: 2, Evaluated: 5, LastProgress: when.Time, CandidateNames: []string{"x", "y"}} + if got := observedStatus(run); !reflect.DeepEqual(got, want) { + t.Fatalf("got %+v, want %+v", got, want) + } + if got := observedStatus(&v1beta2.DynamoGraphDeploymentRun{}); got.HasProgress || !got.LastProgress.IsZero() { + t.Fatalf("an empty status must not look like reported progress: %+v", got) + } +} + +func TestCacheOptionsCoverOnlyThisRunItsCandidatesAndItsPod(t *testing.T) { + cfg := Config{Namespace: "ns", RunName: "run", RunUID: "uid-1", PodName: "pod-1"} + opts := cacheOptions(cfg) + if _, ok := opts.DefaultNamespaces["ns"]; !ok || len(opts.DefaultNamespaces) != 1 { + t.Fatalf("cache must be limited to the run's namespace: %v", opts.DefaultNamespaces) + } + if len(opts.ByObject) != 3 { + t.Fatalf("want selectors for the run, its candidates and the pod, got %d", len(opts.ByObject)) + } + for object, selector := range opts.ByObject { + switch object.(type) { + case *v1beta2.DynamoGraphDeploymentRun: + if selector.Field.String() != "metadata.name=run" { + t.Errorf("run selector = %q", selector.Field) + } + case *v1beta2.DynamoGraphDeploymentCandidate: + if selector.Label.String() != LabelRunUID+"=uid-1" { + t.Errorf("candidate selector = %q", selector.Label) + } + case *corev1.Pod: + if selector.Field.String() != "metadata.name=pod-1" { + t.Errorf("pod selector = %q", selector.Field) + } + default: + t.Errorf("unexpected cached type %T", object) + } + } +} diff --git a/deploy/operator/internal/dgdrrunpublisher/snapshot_test.go b/deploy/operator/internal/dgdrrunpublisher/snapshot_test.go new file mode 100644 index 000000000000..3be65aa92092 --- /dev/null +++ b/deploy/operator/internal/dgdrrunpublisher/snapshot_test.go @@ -0,0 +1,158 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dgdrrunpublisher + +import ( + "encoding/json" + "errors" + "os" + "path/filepath" + "strings" + "testing" +) + +const validManifest = "apiVersion: nvidia.com/v1beta1\nkind: DynamoGraphDeployment\nspec: {}\n" + +const testRunUID = "uid-1" + +func cn(id string) string { return CandidateName("run", testRunUID, id) } + +type cand struct { + id string + failed bool + metrics map[string]any +} + +// snapshotJSON renders a snapshot (JSON is valid YAML, which the real parser accepts). +func snapshotJSON(t *testing.T, phase string, cands ...cand) []byte { + t.Helper() + var list []map[string]any + for _, c := range cands { + entry := map[string]any{"id": c.id, "parameters": map[string]any{"tp": 2}, "metrics": c.metrics} + if c.failed { + entry["outcome"] = OutcomeMaterializationFailed + entry["error"] = "boom" + } else { + entry["outcome"] = OutcomeMaterialized + entry["manifest"] = validManifest + } + list = append(list, entry) + } + data, err := json.Marshal(map[string]any{ + "schemaVersion": 1, + "run": map[string]any{"phase": phase, "terminal": phase != PhaseRunning}, + "progress": map[string]any{"round": 3, "evaluated": len(cands)}, + "candidates": list, + }) + if err != nil { + t.Fatal(err) + } + return data +} + +func writeSnapshot(t *testing.T, dir string, data []byte) { + t.Helper() + if err := os.WriteFile(filepath.Join(dir, SnapshotFileName), data, 0o600); err != nil { + t.Fatal(err) + } +} + +func mustParse(t *testing.T, data []byte) *Snapshot { + t.Helper() + snap, err := ParseSnapshot(data) + if err != nil { + t.Fatal(err) + } + return snap +} + +func withTimestamp(t *testing.T, data []byte, timestamp string) []byte { + t.Helper() + var doc map[string]any + if err := json.Unmarshal(data, &doc); err != nil { + t.Fatal(err) + } + doc["timestamp"] = timestamp + out, err := json.Marshal(doc) + if err != nil { + t.Fatal(err) + } + return out +} + +func TestParseSnapshotValidation(t *testing.T) { + bad := map[string]string{ + "schema": `{"schemaVersion":2,"run":{"phase":"Running","terminal":false}}`, + "phase": `{"schemaVersion":1,"run":{"phase":"Weird","terminal":false}}`, + "terminal flag": `{"schemaVersion":1,"run":{"phase":"Succeeded","terminal":false}}`, + "duplicate": `{"schemaVersion":1,"run":{"phase":"Running","terminal":false},"candidates":[{"id":"a","outcome":"materialized","manifest":"apiVersion: nvidia.com/v1beta1\\nkind: DynamoGraphDeployment\\nspec: {}\\n"},{"id":"a","outcome":"materialized","manifest":"apiVersion: nvidia.com/v1beta1\\nkind: DynamoGraphDeployment\\nspec: {}\\n"}]}`, + "no manifest": `{"schemaVersion":1,"run":{"phase":"Running","terminal":false},"candidates":[{"id":"a","outcome":"materialized"}]}`, + "no error": `{"schemaVersion":1,"run":{"phase":"Running","terminal":false},"candidates":[{"id":"a","outcome":"materialization_failed"}]}`, + "unsafe id": `{"schemaVersion":1,"run":{"phase":"Running","terminal":false},"candidates":[{"id":"Not_Valid","outcome":"materialized","manifest":"apiVersion: nvidia.com/v1beta1\\nkind: DynamoGraphDeployment\\nspec: {}\\n"}]}`, + "long id": `{"schemaVersion":1,"run":{"phase":"Running","terminal":false},"candidates":[{"id":"%s","outcome":"materialized","manifest":"apiVersion: nvidia.com/v1beta1\\nkind: DynamoGraphDeployment\\nspec: {}\\n"}]}`, + "wrong kind": `{"schemaVersion":1,"run":{"phase":"Running","terminal":false},"candidates":[{"id":"a","outcome":"materialized","manifest":"kind: ConfigMap\\nspec: {}\\n"}]}`, + "empty id": `{"schemaVersion":1,"run":{"phase":"Running","terminal":false},"candidates":[{"id":"","outcome":"materialized","manifest":"apiVersion: nvidia.com/v1beta1\\nkind: DynamoGraphDeployment\\nspec: {}\\n"}]}`, + } + bad["long id"] = strings.Replace(bad["long id"], "%s", strings.Repeat("a", 64), 1) + for name, doc := range bad { + if _, err := ParseSnapshot([]byte(doc)); !errors.Is(err, ErrProtocolViolation) { + t.Errorf("%s: want protocol violation, got %v", name, err) + } + } +} + +func TestParseManifestHandlesCompanionResources(t *testing.T) { + dgd := "apiVersion: nvidia.com/v1beta1\nkind: DynamoGraphDeployment\nspec: {}\n" + cm := "apiVersion: v1\nkind: ConfigMap\nmetadata: {}\n" + for name, manifest := range map[string]string{ + "dgd first": dgd + "---\n" + cm, + "configmap first": "# generated\n---\n" + cm + "---\n" + dgd, + } { + got, err := ParseManifest(manifest) + if err != nil { + t.Fatalf("%s: %v", name, err) + } + if got.DGD["kind"] != "DynamoGraphDeployment" || len(got.Companions) != 1 || !strings.Contains(got.Companions[0], "ConfigMap") { + t.Fatalf("%s: %+v", name, got) + } + } + for name, manifest := range map[string]string{ + "two dgds": dgd + "---\n" + dgd, + "no dgd": cm, + "no spec": "apiVersion: nvidia.com/v1beta1\nkind: DynamoGraphDeployment\n", + "no apiVersion": "kind: DynamoGraphDeployment\nspec: {}\n", + "unsupported apiVersion": "apiVersion: nvidia.com/v9\nkind: DynamoGraphDeployment\nspec: {}\n", + } { + if _, err := ParseManifest(manifest); err == nil { + t.Errorf("%s: want error", name) + } + } +} +func writeProgress(t *testing.T, dir, phase string, round, evaluated int) { + t.Helper() + doc := map[string]any{} + if err := json.Unmarshal(snapshotJSON(t, phase, cand{id: "a"}), &doc); err != nil { + t.Fatal(err) + } + doc["progress"] = map[string]any{"round": round, "evaluated": evaluated} + data, err := json.Marshal(doc) + if err != nil { + t.Fatal(err) + } + writeSnapshot(t, dir, data) +} From c611d98dafc83de4b3c0d0a77b3ed095fb221420 Mon Sep 17 00:00:00 2001 From: devivasudevan <49675305+devivasudevan@users.noreply.github.com> Date: Mon, 12 Oct 2026 17:37:42 +0200 Subject: [PATCH 14/40] fix(dgdr-run-publisher): address review (status SSA with resourceVersion, write-echo handling, reconciler tests, ctrl logger) --- .../operator/cmd/dgdr-run-publisher/main.go | 7 +- .../operator/internal/dgdrrunpublisher/doc.go | 4 + .../internal/dgdrrunpublisher/input.go | 8 +- .../internal/dgdrrunpublisher/manager.go | 25 +- .../internal/dgdrrunpublisher/plan.go | 11 + .../internal/dgdrrunpublisher/plan_test.go | 16 + .../internal/dgdrrunpublisher/reconciler.go | 199 ++--- .../dgdrrunpublisher/reconciler_test.go | 736 ++++++++++++++++++ .../internal/dgdrrunpublisher/writer.go | 235 ++++++ 9 files changed, 1114 insertions(+), 127 deletions(-) create mode 100644 deploy/operator/internal/dgdrrunpublisher/writer.go diff --git a/deploy/operator/cmd/dgdr-run-publisher/main.go b/deploy/operator/cmd/dgdr-run-publisher/main.go index 16f446888347..60e55e6cd48b 100644 --- a/deploy/operator/cmd/dgdr-run-publisher/main.go +++ b/deploy/operator/cmd/dgdr-run-publisher/main.go @@ -25,7 +25,6 @@ import ( "context" "flag" "fmt" - "log" "os" "os/signal" "syscall" @@ -33,6 +32,7 @@ import ( "github.com/ai-dynamo/dynamo/deploy/operator/internal/dgdrrunpublisher" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/log/zap" ) func main() { @@ -41,6 +41,8 @@ func main() { func run() int { var cfg dgdrrunpublisher.Config + logOptions := zap.Options{} + logOptions.BindFlags(flag.CommandLine) flag.StringVar(&cfg.SnapshotDir, "snapshot-dir", "", "directory shared with the Sweeper container") flag.StringVar(&cfg.RunName, "run-name", "", "name of the DynamoGraphDeploymentRun") flag.StringVar(&cfg.RunUID, "run-uid", "", "UID of the DynamoGraphDeploymentRun this pod serves") @@ -48,6 +50,7 @@ func run() int { flag.DurationVar(&cfg.MinPollInterval, "poll-interval", time.Second, "snapshot check interval after a change") flag.DurationVar(&cfg.MaxPollInterval, "max-poll-interval", 5*time.Second, "longest snapshot check interval while the snapshot is unchanged") flag.Parse() + ctrl.SetLogger(zap.New(zap.UseFlagOptions(&logOptions))) cfg.Namespace, cfg.PodName = os.Getenv("POD_NAMESPACE"), os.Getenv("POD_NAME") if cfg.SnapshotDir == "" || cfg.RunName == "" || cfg.RunUID == "" || cfg.Namespace == "" || cfg.PodName == "" { @@ -60,7 +63,7 @@ func run() int { err := dgdrrunpublisher.Run(ctx, ctrl.GetConfigOrDie(), cfg) if err != nil { - log.Printf("publisher: %v", err) + ctrl.Log.WithName("dgdr-run-publisher").Error(err, "publisher stopped") } return dgdrrunpublisher.ExitCode(err) } diff --git a/deploy/operator/internal/dgdrrunpublisher/doc.go b/deploy/operator/internal/dgdrrunpublisher/doc.go index ac6cc837ab8c..3d5a83cbee45 100644 --- a/deploy/operator/internal/dgdrrunpublisher/doc.go +++ b/deploy/operator/internal/dgdrrunpublisher/doc.go @@ -34,6 +34,10 @@ // where they differ, and finally deletes obsolete candidates. Failed writes are // retried by the workqueue with backoff. // +// The publisher's own writes come back as watch events and the cache can trail them, so +// the Reconciler remembers what it has written and ignores candidate update events; see +// Reconciler. +// // The manager's cache covers only this Run, its candidates (by the run-UID label) and the // publisher's own Pod, so there is no API-server polling and no spec comparison: // candidate specs are immutable once created. diff --git a/deploy/operator/internal/dgdrrunpublisher/input.go b/deploy/operator/internal/dgdrrunpublisher/input.go index eac2093a2725..9d3183638a8a 100644 --- a/deploy/operator/internal/dgdrrunpublisher/input.go +++ b/deploy/operator/internal/dgdrrunpublisher/input.go @@ -20,7 +20,6 @@ package dgdrrunpublisher import ( "context" "errors" - "log" "os" "reflect" "sync" @@ -105,6 +104,9 @@ func checkProgress(previous, next *Snapshot) error { type Observer struct { Store *InputStore Path string + // OnError reports a problem reading the snapshot, once per distinct message. It is + // optional; the manager wires it to the controller-runtime logger. + OnError func(err error) // Enqueue asks the controller to reconcile the run. It must not block. Enqueue func() // MinInterval is the check interval after a change; it doubles while the input is @@ -157,7 +159,9 @@ func (o *Observer) noteError(err error) { return } o.lastErr = err.Error() - log.Printf("reading snapshot: %v", err) + if o.OnError != nil { + o.OnError(err) + } } // NextInterval is the delay before the next check: it resets to min after a change and diff --git a/deploy/operator/internal/dgdrrunpublisher/manager.go b/deploy/operator/internal/dgdrrunpublisher/manager.go index 2ca8d2c04b84..ffd38e7d20f5 100644 --- a/deploy/operator/internal/dgdrrunpublisher/manager.go +++ b/deploy/operator/internal/dgdrrunpublisher/manager.go @@ -34,6 +34,7 @@ import ( clientgoscheme "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller" @@ -41,6 +42,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/manager" metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" + "sigs.k8s.io/controller-runtime/pkg/predicate" "sigs.k8s.io/controller-runtime/pkg/reconcile" "sigs.k8s.io/controller-runtime/pkg/source" ) @@ -102,6 +104,15 @@ func cacheOptions(c Config) cache.Options { } } +// ignoreCandidateUpdates drops candidate update events. The publisher is the only writer +// of a candidate: its spec is immutable and its status is written once, so an update is +// the echo of our own write and carries nothing to act on. Creates (including the +// initial list) and deletes still enqueue the run key, so a deleted candidate is +// repaired. +var ignoreCandidateUpdates = predicate.Funcs{ + UpdateFunc: func(event.UpdateEvent) bool { return false }, +} + // Run starts the controller manager and returns the publisher's outcome (see the package // comment for its meaning and ExitCode for the exit code), or ctx's error if the process // is told to stop first. @@ -133,6 +144,9 @@ func Run(ctx context.Context, restConfig *rest.Config, cfg Config) error { Path: SnapshotPath(cfg.SnapshotDir), MinInterval: cfg.MinPollInterval, MaxInterval: cfg.MaxPollInterval, + OnError: func(err error) { + ctrl.Log.WithName("input-observer").Error(err, "reading snapshot") + }, Enqueue: func() { // One pending event is enough: the workqueue deduplicates by key anyway. select { @@ -143,7 +157,14 @@ func Run(ctx context.Context, restConfig *rest.Config, cfg Config) error { } completion := NewCompletion() reconciler := &Reconciler{ - Client: mgr.GetClient(), + Client: mgr.GetClient(), + Writer: &clientWriter{ + Client: mgr.GetClient(), + Reader: mgr.GetAPIReader(), + Namespace: cfg.Namespace, + RunName: cfg.RunName, + RunUID: cfg.RunUID, + }, Namespace: cfg.Namespace, RunName: cfg.RunName, RunUID: cfg.RunUID, @@ -160,7 +181,7 @@ func Run(ctx context.Context, restConfig *rest.Config, cfg Config) error { if err := ctrl.NewControllerManagedBy(mgr). Named("dgdr-run-publisher"). For(&v1beta2.DynamoGraphDeploymentRun{}). - Owns(&v1beta2.DynamoGraphDeploymentCandidate{}). + Owns(&v1beta2.DynamoGraphDeploymentCandidate{}, builder.WithPredicates(ignoreCandidateUpdates)). Watches(&corev1.Pod{}, podToRun). WatchesRawSource(source.Channel(events, &handler.EnqueueRequestForObject{})). WithOptions(controller.Options{MaxConcurrentReconciles: 1}). diff --git a/deploy/operator/internal/dgdrrunpublisher/plan.go b/deploy/operator/internal/dgdrrunpublisher/plan.go index ff515b677bed..ec2bc2ee4750 100644 --- a/deploy/operator/internal/dgdrrunpublisher/plan.go +++ b/deploy/operator/internal/dgdrrunpublisher/plan.go @@ -144,6 +144,17 @@ func progressTime(snap *Snapshot) time.Time { return parsed.UTC().Truncate(time.Second) } +// WithPreservedProgress returns the status to write. When the snapshot carries no usable +// timestamp, the progress time already known (observed or written earlier) is kept: +// omitting a field from a server-side-apply payload removes it once this manager owns +// it, so "leave it alone" has to be spelled out as "write the same value again". +func (desired RunStatus) WithPreservedProgress(known time.Time) RunStatus { + if desired.LastProgress.IsZero() { + desired.LastProgress = known + } + return desired +} + // NeedsUpdate reports whether the observed run status differs from the desired one, so // the status is only written when something changed. A zero desired LastProgress means // "leave the observed value alone". diff --git a/deploy/operator/internal/dgdrrunpublisher/plan_test.go b/deploy/operator/internal/dgdrrunpublisher/plan_test.go index a1f05e946d06..f9c24b01befd 100644 --- a/deploy/operator/internal/dgdrrunpublisher/plan_test.go +++ b/deploy/operator/internal/dgdrrunpublisher/plan_test.go @@ -121,3 +121,19 @@ func TestNeedsUpdateOnlyWhenSomethingDiffers(t *testing.T) { } } } + +func TestWithPreservedProgressKeepsTheKnownTimeOnlyWhenTheSnapshotHasNone(t *testing.T) { + known := time.Date(2026, 10, 5, 21, 35, 12, 0, time.UTC) + without := desire(t, withTimestamp(t, snapshotJSON(t, PhaseRunning, cand{id: "a"}), "")).Status + if got := without.WithPreservedProgress(known).LastProgress; !got.Equal(known) { + t.Fatalf("a snapshot without a timestamp must keep the known time, got %v", got) + } + newer := time.Date(2026, 10, 5, 21, 40, 0, 0, time.UTC) + with := desire(t, withTimestamp(t, snapshotJSON(t, PhaseRunning, cand{id: "a"}), "2026-10-05T21:40:00Z")).Status + if got := with.WithPreservedProgress(known).LastProgress; !got.Equal(newer) { + t.Fatalf("a snapshot with a timestamp must win, got %v", got) + } + if got := without.WithPreservedProgress(time.Time{}).LastProgress; !got.IsZero() { + t.Fatalf("nothing known, nothing to preserve, got %v", got) + } +} diff --git a/deploy/operator/internal/dgdrrunpublisher/reconciler.go b/deploy/operator/internal/dgdrrunpublisher/reconciler.go index 184a91d3492b..68c916f1970a 100644 --- a/deploy/operator/internal/dgdrrunpublisher/reconciler.go +++ b/deploy/operator/internal/dgdrrunpublisher/reconciler.go @@ -19,10 +19,8 @@ package dgdrrunpublisher import ( "context" - "encoding/json" "errors" "fmt" - "log" "strings" "time" @@ -31,14 +29,13 @@ import ( "github.com/ai-dynamo/dynamo/deploy/operator/internal/dgdcreconcile" v1beta2 "github.com/ai-dynamo/dynamo/deploy/operator/internal/dgdrrunpublisher/placeholderapi" corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" ) const ( @@ -68,13 +65,28 @@ const ( // Reconciler reconciles the one DynamoGraphDeploymentRun this publisher serves. // // Client is the manager's client: reads come from the scoped informer cache (this Run, -// its candidates, this Pod), writes go to the API server. It needs RBAC (namespaced Role -// bound to the run's ServiceAccount): runs get/list/watch and runs/status patch (server- -// side apply); candidates get/list/watch/create/patch/delete and candidates/status patch; -// pods get/list/watch (own pod, for Sweeper crash detection; resourceNames cannot be used -// with list/watch of generated Job pod names, so the cache narrows with a field selector). +// its candidates, this Pod). Writes go through Writer to the API server. It needs RBAC +// (namespaced Role bound to the run's ServiceAccount): runs get/list/watch and +// runs/status patch (server-side apply); candidates get/list/watch/create/patch/delete +// and candidates/status patch; pods get/list/watch (own pod, for Sweeper crash +// detection; the cache narrows list/watch to it with a field selector because generated +// Job pod names cannot be listed in resourceNames). +// +// Write echoes and cache lag: the publisher's own writes come back as watch events, and +// the cache can trail them. The reconciler therefore remembers what it has written (the +// candidates it created and populated, the last run status it applied, the latest +// progress time) and does not redo that work while the cache catches up. A retry after a +// partial success repeats only the unfinished writes, and an event that merely echoes a +// write finds nothing to do and produces no further write, so it cannot start a loop. +// +// Stale reads: every status write is constrained to the resourceVersion the decision was +// based on (the cached run, the cached candidate, or the version read back from the API +// server for a candidate created a moment ago). A decision made from an older cache +// observation fails with a conflict instead of overwriting newer state, and is retried +// once the informer has delivered the newer version. type Reconciler struct { Client client.Client + Writer writer Namespace string RunName string RunUID string @@ -85,9 +97,17 @@ type Reconciler struct { Store *InputStore Completion *Completion - // finalFailures counts failed reconciles after the Sweeper exited. The controller - // runs one worker, so it needs no locking. + // The fields below are touched only by Reconcile; the controller runs one worker, so + // they need no locking. + + // finalFailures counts failed reconciles after the Sweeper exited. finalFailures int + // written holds the candidates this process has created and populated. + written map[string]struct{} + // applied is the last run status written successfully. + applied *RunStatus + // lastProgress is the latest progress time written. + lastProgress time.Time } func (r *Reconciler) key() types.NamespacedName { @@ -119,7 +139,7 @@ func (r *Reconciler) reconcile(ctx context.Context) error { // exit, so an exit observed here means that write is visible to the check below. state, stateErr := r.sweeperState(ctx) if stateErr != nil { - log.Printf("reading sweeper container state: %v", stateErr) + ctrl.LoggerFrom(ctx).Error(stateErr, "reading sweeper container state") } exited := stateErr == nil && state.Exited if exited { @@ -162,6 +182,11 @@ func (r *Reconciler) retryOrFail(exited bool, err error) error { r.Completion.Finish(err) // retrying cannot fix it return nil } + if apierrors.IsConflict(err) { + // A write was based on a stale cached read. The informer delivers the newer + // version and the retry decides again, so this is not a failed attempt. + return err + } if exited { r.finalFailures++ if r.finalFailures >= finalReconcileAttempts { @@ -182,6 +207,7 @@ func (r *Reconciler) apply(ctx context.Context, run *v1beta2.DynamoGraphDeployme } current := make([]dgdcreconcile.CurrentDGDC, 0, len(list.Items)) uids := make(map[string]types.UID, len(list.Items)) + versions := make(map[string]string, len(list.Items)) // cached resourceVersion per candidate for i := range list.Items { item := &list.Items[i] current = append(current, dgdcreconcile.CurrentDGDC{ @@ -190,130 +216,78 @@ func (r *Reconciler) apply(ctx context.Context, run *v1beta2.DynamoGraphDeployme Incomplete: !meta.IsStatusConditionTrue(item.Status.Conditions, evaluatedCondition), }) uids[item.Name] = item.UID + versions[item.Name] = item.ResourceVersion } actions, err := dgdcreconcile.ComputeActions(desired.Candidates, current) if err != nil { return err } + if r.written == nil { + r.written = map[string]struct{}{} + } for _, candidate := range actions.Creates { - if err := r.createCandidate(ctx, run, desired.Names[candidate.ID], candidate); err != nil { + name := desired.Names[candidate.ID] + if _, done := r.written[name]; done { + continue // created and populated by this process; the cache has not caught up yet + } + version, err := r.Writer.CreateCandidate(ctx, run, name, candidate) + if err != nil { return fmt.Errorf("creating candidate %s: %w", candidate.ID, err) } + versions[name] = version // a new candidate is not in the cache; use the version the API server returned } unpopulated := make([]dgdcreconcile.DesiredCandidate, 0, len(actions.Creates)+len(actions.Incomplete)) unpopulated = append(unpopulated, actions.Creates...) unpopulated = append(unpopulated, actions.Incomplete...) for _, candidate := range unpopulated { - if err := r.populateStatus(ctx, desired.Names[candidate.ID], candidate); err != nil { + name := desired.Names[candidate.ID] + if _, done := r.written[name]; done { + continue + } + if err := r.Writer.PopulateStatus(ctx, name, versions[name], candidate); err != nil { return fmt.Errorf("populating status of candidate %s: %w", candidate.ID, err) } + r.written[name] = struct{}{} } - if desired.Status.NeedsUpdate(observedStatus(run)) { - if err := r.updateRunStatus(ctx, desired.Status); err != nil { - return fmt.Errorf("updating run status: %w", err) - } + if err := r.updateRunStatus(ctx, run, desired.Status); err != nil { + return err } for _, name := range actions.Deletes { - uid := uids[name] - object := &v1beta2.DynamoGraphDeploymentCandidate{ObjectMeta: metav1.ObjectMeta{Namespace: r.Namespace, Name: name}} - if err := client.IgnoreNotFound(r.Client.Delete(ctx, object, client.Preconditions{UID: &uid})); err != nil { + if err := r.Writer.DeleteCandidate(ctx, name, uids[name]); err != nil { return fmt.Errorf("deleting candidate %s: %w", name, err) } } return nil } -// createCandidate creates the candidate with server-side apply. It is only called for a -// candidate that is missing from the cache, and re-applying the same content after a -// stale cache read is a no-op, so no spec comparison is needed. -func (r *Reconciler) createCandidate(ctx context.Context, run *v1beta2.DynamoGraphDeploymentRun, name string, candidate dgdcreconcile.DesiredCandidate) error { - spec, companions, err := decodeManifest(candidate) - if err != nil { - return err - } - parameters, err := normalizeJSON(candidate.Parameters) - if err != nil { - return err - } - object := &v1beta2.DynamoGraphDeploymentCandidate{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: r.Namespace, - Name: name, - Labels: map[string]string{LabelRunUID: r.RunUID, LabelCandidateID: candidate.ID}, - }, - Spec: v1beta2.DynamoGraphDeploymentCandidateSpec{DynamoGraphDeploymentSpec: *spec}, - } - if companions != "" { - object.Annotations = map[string]string{AnnotationAdditionalResources: companions} +// updateRunStatus writes the run status when it differs from what the cache shows and +// from what this process already wrote. A snapshot without a usable timestamp keeps the +// latest progress time known, because a server-side-apply payload that omits the field +// would remove it. +func (r *Reconciler) updateRunStatus(ctx context.Context, run *v1beta2.DynamoGraphDeploymentRun, desired RunStatus) error { + observed := observedStatus(run) + if !desired.NeedsUpdate(observed) { + return nil } - if err := controllerutil.SetControllerReference(run, object, r.Client.Scheme()); err != nil { - return err + known := observed.LastProgress + if r.lastProgress.After(known) { + known = r.lastProgress } - content, err := runtime.DefaultUnstructuredConverter.ToUnstructured(object) - if err != nil { - return err + write := desired.WithPreservedProgress(known) + if r.applied != nil && !write.NeedsUpdate(*r.applied) { + return nil // already written; the cache has not caught up yet } - // The status is written separately, through the status subresource. - delete(content, "status") - unstructured.RemoveNestedField(content, "metadata", "creationTimestamp") - if err := unstructured.SetNestedField(content, parameters, "spec", "parameters"); err != nil { - return err + if err := r.Writer.UpdateRunStatus(ctx, run.ResourceVersion, write); err != nil { + return fmt.Errorf("updating run status: %w", err) } - apply := &unstructured.Unstructured{Object: content} - apply.SetGroupVersionKind(v1beta2.GroupVersion.WithKind("DynamoGraphDeploymentCandidate")) - return r.Client.Apply(ctx, client.ApplyConfigurationFromUnstructured(apply), client.FieldOwner(fieldOwner), client.ForceOwnership) -} - -// populateStatus records the evaluation facts once. The candidate is referenced from the -// run only after this, and its status is immutable afterwards. -func (r *Reconciler) populateStatus(ctx context.Context, name string, candidate dgdcreconcile.DesiredCandidate) error { - metrics, err := normalizeJSON(candidate.Metrics) - if err != nil { - return err + r.applied = &write + if write.LastProgress.After(r.lastProgress) { + r.lastProgress = write.LastProgress } - apply := &unstructured.Unstructured{Object: map[string]any{ - "metadata": map[string]any{"namespace": r.Namespace, "name": name}, - "status": map[string]any{ - "metrics": metrics, - "conditions": []any{map[string]any{ - "type": evaluatedCondition, - "status": string(metav1.ConditionTrue), - "reason": evaluatedCondition, - "message": "evaluated by the Sweeper", - "lastTransitionTime": time.Now().UTC().Format(time.RFC3339), - }}, - }, - }} - apply.SetGroupVersionKind(v1beta2.GroupVersion.WithKind("DynamoGraphDeploymentCandidate")) - return r.Client.Status().Apply(ctx, client.ApplyConfigurationFromUnstructured(apply), client.FieldOwner(fieldOwner), client.ForceOwnership) -} - -// updateRunStatus writes the publisher-owned run status fields with server-side apply, so -// the run controller's own status fields (conditions) are left alone. -func (r *Reconciler) updateRunStatus(ctx context.Context, status RunStatus) error { - refs := make([]any, 0, len(status.CandidateNames)) - for _, name := range status.CandidateNames { - refs = append(refs, map[string]any{"name": name}) - } - fields := map[string]any{ - "message": status.Message, - "candidateRefs": refs, - } - if status.HasProgress { - fields["progress"] = map[string]any{"round": int64(status.Round), "evaluated": int64(status.Evaluated)} - } - if !status.LastProgress.IsZero() { - fields["lastProgressTime"] = status.LastProgress.UTC().Format(time.RFC3339) - } - apply := &unstructured.Unstructured{Object: map[string]any{ - "metadata": map[string]any{"namespace": r.Namespace, "name": r.RunName}, - "status": fields, - }} - apply.SetGroupVersionKind(v1beta2.GroupVersion.WithKind("DynamoGraphDeploymentRun")) - return r.Client.Status().Apply(ctx, client.ApplyConfigurationFromUnstructured(apply), client.FieldOwner(fieldOwner), client.ForceOwnership) + return nil } // observedStatus is the publisher-owned part of the run's current (cached) status. @@ -355,23 +329,6 @@ func sweeperStateOf(pod *corev1.Pod, container string) SweeperState { return SweeperState{} } -// normalizeJSON round-trips a value through JSON so it only holds the types unstructured -// objects allow (int64, float64, string, bool, maps, slices). -func normalizeJSON(value map[string]any) (map[string]any, error) { - out := map[string]any{} - if value == nil { - return out, nil - } - raw, err := json.Marshal(value) - if err != nil { - return nil, err - } - if err := json.Unmarshal(raw, &out); err != nil { - return nil, err - } - return out, nil -} - // decodeManifest extracts the DGD spec and the companion resources from a rendered // manifest. A DGD in either supported apiVersion is converted to v1beta1, the same way // the DGDR controller does it. diff --git a/deploy/operator/internal/dgdrrunpublisher/reconciler_test.go b/deploy/operator/internal/dgdrrunpublisher/reconciler_test.go index 52467339ab07..8387962b2ff6 100644 --- a/deploy/operator/internal/dgdrrunpublisher/reconciler_test.go +++ b/deploy/operator/internal/dgdrrunpublisher/reconciler_test.go @@ -18,15 +18,587 @@ package dgdrrunpublisher import ( + "context" + "encoding/json" + "errors" + "path/filepath" "reflect" + "strings" "testing" "time" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/dgdcreconcile" v1beta2 "github.com/ai-dynamo/dynamo/deploy/operator/internal/dgdrrunpublisher/placeholderapi" corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" ) +const ( + testNamespace = "ns" + testRunName = "run" + testPodName = "pod-1" +) + +// memWriter is the reconciler's writer for tests. It logs every write attempt (a failed +// attempt is logged with a trailing "!") and, unless lag is set, applies the write to +// the fake client the reconciler reads from, which stands in for the informer cache. +// With lag set, writes are logged but never become visible, like a cache that has not +// caught up. +type memWriter struct { + t *testing.T + c client.Client + ops []string + status []RunStatus + // statusVersions and populateVersions record the resourceVersion each write was + // constrained to. + statusVersions []string + populateVersions map[string]string + lag bool + // failOn makes the named operation ("create", "populate", "status", "delete") fail; + // once, unless sticky. + failOn map[string]error + sticky bool +} + +func (w *memWriter) fail(op string) error { + err, ok := w.failOn[op] + if !ok { + return nil + } + if !w.sticky { + delete(w.failOn, op) + } + return err +} + +func (w *memWriter) record(op string, err error) error { + if err != nil { + op += "!" + } + w.ops = append(w.ops, op) + return err +} + +func (w *memWriter) CreateCandidate(ctx context.Context, _ *v1beta2.DynamoGraphDeploymentRun, name string, candidate dgdcreconcile.DesiredCandidate) (string, error) { + if err := w.record("create:"+name, w.fail("create")); err != nil { + return "", err + } + version := createdVersion(name) + if w.lag { + return version, nil + } + object := &v1beta2.DynamoGraphDeploymentCandidate{ObjectMeta: metav1.ObjectMeta{ + Namespace: testNamespace, + Name: name, + UID: types.UID("uid-" + name), + Labels: map[string]string{LabelRunUID: testRunUID, LabelCandidateID: candidate.ID}, + }} + return version, client.IgnoreAlreadyExists(w.c.Create(ctx, object)) +} + +// createdVersion is the resourceVersion the fake API server "returns" for a new candidate. +func createdVersion(name string) string { return "created-" + name } + +func (w *memWriter) PopulateStatus(ctx context.Context, name, resourceVersion string, _ dgdcreconcile.DesiredCandidate) error { + if err := w.record("populate:"+name, w.fail("populate")); err != nil { + return err + } + w.populateVersions[name] = resourceVersion + if w.lag { + return nil + } + var object v1beta2.DynamoGraphDeploymentCandidate + if err := w.c.Get(ctx, types.NamespacedName{Namespace: testNamespace, Name: name}, &object); err != nil { + return err + } + meta.SetStatusCondition(&object.Status.Conditions, metav1.Condition{Type: evaluatedCondition, Status: metav1.ConditionTrue, Reason: evaluatedCondition}) + return w.c.Update(ctx, &object) +} + +func (w *memWriter) UpdateRunStatus(ctx context.Context, resourceVersion string, status RunStatus) error { + if err := w.record("status", w.fail("status")); err != nil { + return err + } + w.status = append(w.status, status) + w.statusVersions = append(w.statusVersions, resourceVersion) + if w.lag { + return nil + } + var run v1beta2.DynamoGraphDeploymentRun + if err := w.c.Get(ctx, types.NamespacedName{Namespace: testNamespace, Name: testRunName}, &run); err != nil { + return err + } + run.Status.Message = status.Message + run.Status.Progress = &v1beta2.RunProgress{Round: status.Round, Evaluated: status.Evaluated} + if !status.LastProgress.IsZero() { + progressed := metav1.NewTime(status.LastProgress) + run.Status.LastProgressTime = &progressed + } + run.Status.CandidateRefs = nil + for _, name := range status.CandidateNames { + run.Status.CandidateRefs = append(run.Status.CandidateRefs, v1beta2.CandidateRef{Name: name}) + } + return w.c.Update(ctx, &run) +} + +func (w *memWriter) DeleteCandidate(ctx context.Context, name string, _ types.UID) error { + if err := w.record("delete:"+name, w.fail("delete")); err != nil { + return err + } + if w.lag { + return nil + } + return client.IgnoreNotFound(w.c.Delete(ctx, &v1beta2.DynamoGraphDeploymentCandidate{ObjectMeta: metav1.ObjectMeta{Namespace: testNamespace, Name: name}})) +} + +type fixture struct { + t *testing.T + r *Reconciler + w *memWriter + c client.Client +} + +func newFixture(t *testing.T) *fixture { + t.Helper() + scheme, err := NewScheme() + if err != nil { + t.Fatal(err) + } + run := &v1beta2.DynamoGraphDeploymentRun{ObjectMeta: metav1.ObjectMeta{Namespace: testNamespace, Name: testRunName, UID: testRunUID}} + pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Namespace: testNamespace, Name: testPodName}} + c := fake.NewClientBuilder().WithScheme(scheme).WithObjects(run, pod).Build() + store := &InputStore{} + observer := &Observer{ + Store: store, + Path: filepath.Join(t.TempDir(), SnapshotFileName), + MinInterval: time.Millisecond, + MaxInterval: time.Millisecond, + } + w := &memWriter{t: t, c: c, failOn: map[string]error{}, populateVersions: map[string]string{}} + return &fixture{t: t, c: c, w: w, r: &Reconciler{ + Client: c, + Writer: w, + Namespace: testNamespace, + RunName: testRunName, + RunUID: testRunUID, + PodName: testPodName, + SweeperContainer: "sweeper", + Observer: observer, + Store: store, + Completion: NewCompletion(), + }} +} + +// publish writes a snapshot the way the Sweeper does and lets the observer see it. +func (f *fixture) publish(data []byte) { + f.t.Helper() + writeSnapshot(f.t, filepath.Dir(f.r.Observer.Path), data) + f.r.Observer.Check() +} + +func (f *fixture) reconcile() error { + f.t.Helper() + _, err := f.r.Reconcile(context.Background(), ctrl.Request{NamespacedName: types.NamespacedName{Namespace: testNamespace, Name: testRunName}}) + return err +} + +func (f *fixture) mustReconcile() { + f.t.Helper() + if err := f.reconcile(); err != nil { + f.t.Fatalf("reconcile: %v", err) + } +} + +// exitSweeper reports the Sweeper container as terminated, as the pod status does. +func (f *fixture) exitSweeper(code int32) { + f.t.Helper() + var pod corev1.Pod + if err := f.c.Get(context.Background(), types.NamespacedName{Namespace: testNamespace, Name: testPodName}, &pod); err != nil { + f.t.Fatal(err) + } + pod.Status.ContainerStatuses = []corev1.ContainerStatus{{Name: "sweeper", State: corev1.ContainerState{Terminated: &corev1.ContainerStateTerminated{ExitCode: code}}}} + if err := f.c.Status().Update(context.Background(), &pod); err != nil { + f.t.Fatal(err) + } +} + +func (f *fixture) takeOps() []string { + ops := f.w.ops + f.w.ops = nil + return ops +} + +func (f *fixture) lastStatus() RunStatus { + f.t.Helper() + if len(f.w.status) == 0 { + f.t.Fatal("no run status was written") + } + return f.w.status[len(f.w.status)-1] +} + +// withProgress rewrites a snapshot's progress counters. +func withProgress(t *testing.T, data []byte, round, evaluated int) []byte { + t.Helper() + doc := map[string]any{} + if err := json.Unmarshal(data, &doc); err != nil { + t.Fatal(err) + } + doc["progress"] = map[string]any{"round": round, "evaluated": evaluated} + out, err := json.Marshal(doc) + if err != nil { + t.Fatal(err) + } + return out +} + +func TestReconcileCreatesPopulatesAndReferencesCandidatesInOrder(t *testing.T) { + f := newFixture(t) + f.publish(snapshotJSON(t, PhaseRunning, cand{id: "a"}, cand{id: "b"})) + f.mustReconcile() + want := []string{"create:" + cn("a"), "create:" + cn("b"), "populate:" + cn("a"), "populate:" + cn("b"), "status"} + if got := f.takeOps(); !reflect.DeepEqual(got, want) { + t.Fatalf("ops = %v, want %v", got, want) + } + if got, want := f.lastStatus().CandidateNames, []string{cn("a"), cn("b")}; !reflect.DeepEqual(got, want) { + t.Fatalf("refs = %v, want %v", got, want) + } +} + +func TestSwapOnlyReordersTheReferences(t *testing.T) { + f := newFixture(t) + f.publish(snapshotJSON(t, PhaseRunning, cand{id: "a"}, cand{id: "b"})) + f.mustReconcile() + f.takeOps() + f.publish(snapshotJSON(t, PhaseRunning, cand{id: "b"}, cand{id: "a"})) + f.mustReconcile() + if got := f.takeOps(); !reflect.DeepEqual(got, []string{"status"}) { + t.Fatalf("a swap must not touch any candidate, ops = %v", got) + } + if got, want := f.lastStatus().CandidateNames, []string{cn("b"), cn("a")}; !reflect.DeepEqual(got, want) { + t.Fatalf("refs = %v, want %v", got, want) + } +} + +func TestReconcilingUnchangedStateWritesNothing(t *testing.T) { + f := newFixture(t) + f.publish(snapshotJSON(t, PhaseRunning, cand{id: "a"})) + f.mustReconcile() + f.takeOps() + f.mustReconcile() + f.mustReconcile() + if got := f.takeOps(); len(got) != 0 { + t.Fatalf("unchanged state must cause no writes, got %v", got) + } +} + +func TestCacheLagDoesNotRepeatWrites(t *testing.T) { + f := newFixture(t) + f.w.lag = true // the cache never shows what was written + f.publish(snapshotJSON(t, PhaseRunning, cand{id: "a"})) + f.mustReconcile() + first := f.takeOps() + if len(first) != 3 { + t.Fatalf("first reconcile ops = %v", first) + } + f.mustReconcile() + f.mustReconcile() + if got := f.takeOps(); len(got) != 0 { + t.Fatalf("an echo-triggered reconcile must not redo writes the cache has not shown yet, got %v", got) + } +} + +func TestPartialSuccessRetryRepeatsOnlyTheUnfinishedWrites(t *testing.T) { + f := newFixture(t) + f.w.lag = true + f.w.failOn["status"] = errors.New("api unavailable") + f.publish(snapshotJSON(t, PhaseRunning, cand{id: "a"})) + if err := f.reconcile(); err == nil { + t.Fatal("a failed run status write must be returned for the workqueue to retry with backoff") + } + f.mustReconcile() // the retry, possibly triggered early by the echo of the candidate create + want := []string{"create:" + cn("a"), "populate:" + cn("a"), "status!", "status"} + if got := f.takeOps(); !reflect.DeepEqual(got, want) { + t.Fatalf("ops = %v, want %v", got, want) + } +} + +func TestCreateFailureSkipsTheRunStatusAndIsRetried(t *testing.T) { + f := newFixture(t) + f.w.failOn["create"] = errors.New("boom") + f.publish(snapshotJSON(t, PhaseRunning, cand{id: "a"})) + if err := f.reconcile(); err == nil { + t.Fatal("want the create failure returned") + } + for _, op := range f.takeOps() { + if op == "status" { + t.Fatal("the run must not reference a candidate that was not created") + } + } + f.mustReconcile() + if got := f.takeOps(); got[len(got)-1] != "status" { + t.Fatalf("ops after retry = %v", got) + } +} + +func TestDroppedCandidateIsDeletedAfterTheRunStatusUpdate(t *testing.T) { + f := newFixture(t) + f.publish(snapshotJSON(t, PhaseRunning, cand{id: "a"}, cand{id: "b"})) + f.mustReconcile() + f.takeOps() + f.publish(withProgress(t, snapshotJSON(t, PhaseRunning, cand{id: "a"}), 3, 9)) + f.mustReconcile() + if got, want := f.takeOps(), []string{"status", "delete:" + cn("b")}; !reflect.DeepEqual(got, want) { + t.Fatalf("ops = %v, want %v", got, want) + } +} + +func TestFailedMaterializationIsNotCreatedButCounted(t *testing.T) { + f := newFixture(t) + f.publish(snapshotJSON(t, PhaseRunning, cand{id: "a"}, cand{id: "x", failed: true})) + f.mustReconcile() + for _, op := range f.takeOps() { + if strings.Contains(op, "x") && strings.HasPrefix(op, "create") { + t.Fatalf("a failed candidate must not be created: %s", op) + } + } + status := f.lastStatus() + if !reflect.DeepEqual(status.CandidateNames, []string{cn("a")}) || !strings.Contains(status.Message, "1 candidate(s) failed materialization") { + t.Fatalf("status = %+v", status) + } +} + +func TestCandidateWithoutStatusIsCompletedNotRecreated(t *testing.T) { + f := newFixture(t) + existing := &v1beta2.DynamoGraphDeploymentCandidate{ObjectMeta: metav1.ObjectMeta{ + Namespace: testNamespace, + Name: cn("a"), + Labels: map[string]string{LabelRunUID: testRunUID, LabelCandidateID: "a"}, + }} + if err := f.c.Create(context.Background(), existing); err != nil { + t.Fatal(err) + } + f.publish(snapshotJSON(t, PhaseRunning, cand{id: "a"})) + f.mustReconcile() + if got, want := f.takeOps(), []string{"populate:" + cn("a"), "status"}; !reflect.DeepEqual(got, want) { + t.Fatalf("ops = %v, want %v", got, want) + } +} + +func TestProgressTimeSurvivesASnapshotWithoutATimestamp(t *testing.T) { + for name, lag := range map[string]bool{"cache current": false, "cache lagging": true} { + t.Run(name, func(t *testing.T) { + f := newFixture(t) + f.w.lag = lag + first := time.Date(2026, 10, 5, 21, 35, 12, 0, time.UTC) + f.publish(withTimestamp(t, snapshotJSON(t, PhaseRunning, cand{id: "a"}), "2026-10-05T21:35:12Z")) + f.mustReconcile() + f.publish(withTimestamp(t, snapshotJSON(t, PhaseRunning, cand{id: "a"}, cand{id: "b"}), "")) + f.mustReconcile() + if len(f.w.status) != 2 { + t.Fatalf("want two status writes, got %d", len(f.w.status)) + } + if got := f.lastStatus().LastProgress; !got.Equal(first) { + t.Fatalf("the second write must repeat the known progress time %v, got %v", first, got) + } + }) + } +} + +func TestTerminalSucceededSnapshotIsReconciledThenAcknowledged(t *testing.T) { + f := newFixture(t) + f.publish(snapshotJSON(t, PhaseSucceeded, cand{id: "a"})) + f.mustReconcile() + if !f.r.Completion.Finished() || f.r.Completion.Err() != nil { + t.Fatalf("finished=%v err=%v", f.r.Completion.Finished(), f.r.Completion.Err()) + } + if len(f.w.status) != 1 { + t.Fatal("the terminal snapshot must be reconciled before the publisher exits") + } +} + +func TestTerminalFailedSnapshotIsReconciledThenFails(t *testing.T) { + f := newFixture(t) + f.publish(snapshotJSON(t, PhaseFailed, cand{id: "a"})) + f.mustReconcile() + if err := f.r.Completion.Err(); !errors.Is(err, ErrRunFailed) || ExitCode(err) != ExitRunFailed { + t.Fatalf("err = %v", err) + } + if len(f.w.status) != 1 { + t.Fatal("the failed snapshot must still be reconciled") + } +} + +func TestTerminalSnapshotReconcileFailureIsRetriedUntilItSucceeds(t *testing.T) { + f := newFixture(t) + f.w.failOn["status"] = errors.New("api unavailable") + f.publish(snapshotJSON(t, PhaseSucceeded, cand{id: "a"})) + if err := f.reconcile(); err == nil || f.r.Completion.Finished() { + t.Fatalf("a terminal snapshot that could not be reconciled must be retried, err=%v", err) + } + f.mustReconcile() + if !f.r.Completion.Finished() || f.r.Completion.Err() != nil { + t.Fatalf("finished=%v err=%v", f.r.Completion.Finished(), f.r.Completion.Err()) + } +} + +func TestSweeperCrashWithoutTerminalSnapshotReconcilesTheLastSnapshot(t *testing.T) { + f := newFixture(t) + f.publish(snapshotJSON(t, PhaseRunning, cand{id: "a"})) + f.exitSweeper(137) + f.mustReconcile() + if !f.r.Completion.Finished() || f.r.Completion.Err() != nil { + t.Fatalf("the Sweeper's own exit code fails the Job, not the publisher: finished=%v err=%v", f.r.Completion.Finished(), f.r.Completion.Err()) + } + if len(f.w.status) != 1 { + t.Fatal("the last snapshot must be reconciled before exit") + } +} + +func TestSweeperCleanExitWithoutTerminalSnapshotIsAProtocolViolation(t *testing.T) { + f := newFixture(t) + f.publish(snapshotJSON(t, PhaseRunning, cand{id: "a"})) + f.exitSweeper(0) + f.mustReconcile() + if err := f.r.Completion.Err(); !errors.Is(err, ErrMissingTerminalSnapshot) || ExitCode(err) != ExitProtocolViolation { + t.Fatalf("err = %v", err) + } + if len(f.w.status) != 1 { + t.Fatal("the last snapshot must be reconciled before exit") + } +} + +func TestSweeperExitBeforeAnySnapshot(t *testing.T) { + crash := newFixture(t) + crash.exitSweeper(1) + crash.mustReconcile() + if !crash.r.Completion.Finished() || crash.r.Completion.Err() != nil { + t.Fatalf("crash: finished=%v err=%v", crash.r.Completion.Finished(), crash.r.Completion.Err()) + } + clean := newFixture(t) + clean.exitSweeper(0) + clean.mustReconcile() + if err := clean.r.Completion.Err(); !errors.Is(err, ErrMissingTerminalSnapshot) { + t.Fatalf("clean exit with no snapshot at all: err = %v", err) + } +} + +func TestSnapshotWrittenJustBeforeTheSweeperExitIsReadAfterTheExitIsSeen(t *testing.T) { + f := newFixture(t) + f.publish(snapshotJSON(t, PhaseRunning, cand{id: "a"})) + f.mustReconcile() + // The Sweeper writes its terminal snapshot and exits; the observer has not looked yet. + writeSnapshot(t, filepath.Dir(f.r.Observer.Path), snapshotJSON(t, PhaseSucceeded, cand{id: "a"}, cand{id: "b"})) + f.exitSweeper(0) + f.mustReconcile() + if !f.r.Completion.Finished() || f.r.Completion.Err() != nil { + t.Fatalf("the terminal snapshot must be picked up on exit: finished=%v err=%v", f.r.Completion.Finished(), f.r.Completion.Err()) + } + if got := f.lastStatus().CandidateNames; len(got) != 2 { + t.Fatalf("refs = %v", got) + } +} + +func TestInvalidFinalSnapshotIsAProtocolViolationExit(t *testing.T) { + f := newFixture(t) + writeSnapshot(t, filepath.Dir(f.r.Observer.Path), []byte(`{"schemaVersion":2}`)) + f.exitSweeper(0) + f.mustReconcile() + if err := f.r.Completion.Err(); ExitCode(err) != ExitProtocolViolation { + t.Fatalf("err = %v, exit = %d", err, ExitCode(err)) + } +} + +func TestRegressingSnapshotIsNotReconciled(t *testing.T) { + f := newFixture(t) + f.publish(withProgress(t, snapshotJSON(t, PhaseRunning, cand{id: "a"}), 3, 5)) + f.mustReconcile() + f.takeOps() + f.publish(withProgress(t, snapshotJSON(t, PhaseRunning, cand{id: "a"}, cand{id: "b"}), 2, 99)) + f.mustReconcile() + if got := f.takeOps(); len(got) != 0 { + t.Fatalf("a regressing snapshot must not be reconciled, got %v", got) + } +} + +func TestFinalReconcileFailureEndsTheProcessAfterBoundedRetries(t *testing.T) { + f := newFixture(t) + f.w.failOn["status"] = errors.New("api unavailable") + f.w.sticky = true + f.publish(snapshotJSON(t, PhaseRunning, cand{id: "a"})) + f.exitSweeper(1) + for attempt := 1; attempt < finalReconcileAttempts; attempt++ { + if err := f.reconcile(); err == nil || f.r.Completion.Finished() { + t.Fatalf("attempt %d: want a retry, err=%v", attempt, err) + } + } + f.mustReconcile() + if err := f.r.Completion.Err(); err == nil || ExitCode(err) != ExitReconcileFailed { + t.Fatalf("err = %v", err) + } +} + +func TestMalformedProjectionFailsImmediately(t *testing.T) { + f := newFixture(t) + f.publish(snapshotJSON(t, PhaseRunning, cand{id: "a"})) + // Two live candidates claiming the same id make the diff input malformed. + for _, name := range []string{"x-1", "x-2"} { + object := &v1beta2.DynamoGraphDeploymentCandidate{ObjectMeta: metav1.ObjectMeta{ + Namespace: testNamespace, Name: name, + Labels: map[string]string{LabelRunUID: testRunUID, LabelCandidateID: "dup"}, + }} + if err := f.c.Create(context.Background(), object); err != nil { + t.Fatal(err) + } + } + f.mustReconcile() + var inputErr *dgdcreconcile.DiffInputError + if err := f.r.Completion.Err(); !errors.As(err, &inputErr) || ExitCode(err) != ExitReconcileFailed { + t.Fatalf("err = %v", err) + } +} + +func TestRunRecreatedUnderTheSameNameIsNotTouched(t *testing.T) { + f := newFixture(t) + var run v1beta2.DynamoGraphDeploymentRun + if err := f.c.Get(context.Background(), types.NamespacedName{Namespace: testNamespace, Name: testRunName}, &run); err != nil { + t.Fatal(err) + } + run.UID = "another-uid" + if err := f.c.Update(context.Background(), &run); err != nil { + t.Fatal(err) + } + f.publish(snapshotJSON(t, PhaseRunning, cand{id: "a"})) + f.mustReconcile() + if !f.r.Completion.Finished() || f.r.Completion.Err() == nil { + t.Fatal("a publisher serving a previous incarnation must stop") + } + if len(f.w.ops) != 0 { + t.Fatalf("it must not write anything: %v", f.w.ops) + } +} + +func TestReconcileIgnoresOtherKeysAndAFinishedRun(t *testing.T) { + f := newFixture(t) + f.publish(snapshotJSON(t, PhaseSucceeded, cand{id: "a"})) + if _, err := f.r.Reconcile(context.Background(), ctrl.Request{NamespacedName: types.NamespacedName{Namespace: testNamespace, Name: "other"}}); err != nil || len(f.w.ops) != 0 { + t.Fatalf("another key must be ignored: err=%v ops=%v", err, f.w.ops) + } + f.mustReconcile() + ops := len(f.w.ops) + f.mustReconcile() + if len(f.w.ops) != ops { + t.Fatal("a finished run must not be reconciled again") + } +} + func podWith(statuses ...corev1.ContainerStatus) *corev1.Pod { return &corev1.Pod{Status: corev1.PodStatus{ContainerStatuses: statuses}} } @@ -99,3 +671,167 @@ func TestCacheOptionsCoverOnlyThisRunItsCandidatesAndItsPod(t *testing.T) { } } } + +func nested(t *testing.T, object *unstructured.Unstructured, fields ...string) (any, bool) { + t.Helper() + value, found, err := unstructured.NestedFieldNoCopy(object.Object, fields...) + if err != nil { + t.Fatal(err) + } + return value, found +} + +func TestCandidateApplyPayload(t *testing.T) { + scheme, err := NewScheme() + if err != nil { + t.Fatal(err) + } + run := &v1beta2.DynamoGraphDeploymentRun{ObjectMeta: metav1.ObjectMeta{Namespace: testNamespace, Name: testRunName, UID: testRunUID}} + candidate := dgdcreconcile.DesiredCandidate{ID: "a", Spec: validManifest, Parameters: map[string]any{"tp": 2}} + apply, err := buildCandidateApply(scheme, testNamespace, testRunUID, run, cn("a"), candidate) + if err != nil { + t.Fatal(err) + } + if apply.GetAPIVersion() != "nvidia.com/v1beta2" || apply.GetKind() != "DynamoGraphDeploymentCandidate" || apply.GetName() != cn("a") || apply.GetNamespace() != testNamespace { + t.Fatalf("identity = %s %s %s/%s", apply.GetAPIVersion(), apply.GetKind(), apply.GetNamespace(), apply.GetName()) + } + if _, found := nested(t, apply, "status"); found { + t.Fatal("status goes through the status subresource") + } + if _, found := nested(t, apply, "metadata", "creationTimestamp"); found { + t.Fatal("creationTimestamp must not be part of an apply payload") + } + if got := apply.GetLabels(); got[LabelRunUID] != testRunUID || got[LabelCandidateID] != "a" { + t.Fatalf("labels = %v", got) + } + if _, found := nested(t, apply, "spec", "parameters"); !found { + t.Fatal("the resolved parameters belong in the spec") + } + owners := apply.GetOwnerReferences() + if len(owners) != 1 || owners[0].UID != testRunUID || owners[0].Controller == nil || !*owners[0].Controller { + t.Fatalf("owner references = %+v", owners) + } +} + +func TestCandidateStatusApplyPayloadIsDeterministic(t *testing.T) { + candidate := dgdcreconcile.DesiredCandidate{ID: "a", Metrics: map[string]any{"score": 2}} + now := time.Date(2026, 10, 5, 21, 35, 12, 0, time.UTC) + first, err := buildCandidateStatusApply(testNamespace, cn("a"), "17", candidate, now) + if err != nil { + t.Fatal(err) + } + second, err := buildCandidateStatusApply(testNamespace, cn("a"), "17", candidate, now) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(first.Object, second.Object) { + t.Fatal("the same input must produce the same payload") + } + if got, _ := nested(t, first, "metadata", "resourceVersion"); got != "17" { + t.Fatalf("the status write must be constrained to the version it was decided on, got %v", got) + } + conditions, _ := nested(t, first, "status", "conditions") + list, ok := conditions.([]any) + if !ok || len(list) != 1 { + t.Fatalf("conditions = %v", conditions) + } + condition := list[0].(map[string]any) + if condition["type"] != evaluatedCondition || condition["status"] != "True" || condition["lastTransitionTime"] != "2026-10-05T21:35:12Z" { + t.Fatalf("condition = %v", condition) + } +} + +func TestRunStatusApplyPayload(t *testing.T) { + when := time.Date(2026, 10, 5, 21, 35, 12, 0, time.UTC) + status := RunStatus{Message: "m", HasProgress: true, Round: 2, Evaluated: 5, LastProgress: when, CandidateNames: []string{"x", "y"}} + apply := buildRunStatusApply(testNamespace, testRunName, "42", status) + if apply.GetKind() != "DynamoGraphDeploymentRun" || apply.GetName() != testRunName { + t.Fatalf("identity = %s %s", apply.GetKind(), apply.GetName()) + } + if got, _ := nested(t, apply, "metadata", "resourceVersion"); got != "42" { + t.Fatalf("the status write must be constrained to the observed run version, got %v", got) + } + if _, found := nested(t, buildRunStatusApply(testNamespace, testRunName, "", status), "metadata", "resourceVersion"); found { + t.Fatal("no version known, no constraint") + } + if got, _ := nested(t, apply, "status", "lastProgressTime"); got != "2026-10-05T21:35:12Z" { + t.Fatalf("lastProgressTime = %v", got) + } + if got, _ := nested(t, apply, "status", "progress", "round"); got != int64(2) { + t.Fatalf("round = %#v", got) + } + refs, _ := nested(t, apply, "status", "candidateRefs") + if list, ok := refs.([]any); !ok || len(list) != 2 { + t.Fatalf("candidateRefs = %v", refs) + } + // A status without a progress time leaves the field out; the reconciler is what + // carries the known time into the status it writes. + status.LastProgress = time.Time{} + if _, found := nested(t, buildRunStatusApply(testNamespace, testRunName, "42", status), "status", "lastProgressTime"); found { + t.Fatal("no time known, so none is written") + } +} + +func TestRunStatusWriteIsConstrainedToTheObservedRunVersion(t *testing.T) { + f := newFixture(t) + var run v1beta2.DynamoGraphDeploymentRun + if err := f.c.Get(context.Background(), types.NamespacedName{Namespace: testNamespace, Name: testRunName}, &run); err != nil { + t.Fatal(err) + } + if run.ResourceVersion == "" { + t.Fatal("the fake client should assign a resourceVersion") + } + f.publish(snapshotJSON(t, PhaseRunning, cand{id: "a"})) + f.mustReconcile() + if len(f.w.statusVersions) != 1 || f.w.statusVersions[0] != run.ResourceVersion { + t.Fatalf("status versions = %v, want [%s]", f.w.statusVersions, run.ResourceVersion) + } +} + +func TestNewCandidateStatusUsesTheVersionReturnedByTheCreate(t *testing.T) { + f := newFixture(t) + f.publish(snapshotJSON(t, PhaseRunning, cand{id: "a"})) + f.mustReconcile() + if got, want := f.w.populateVersions[cn("a")], createdVersion(cn("a")); got != want { + t.Fatalf("populate version = %q, want the version from the create (%q)", got, want) + } +} + +func TestExistingCandidateStatusUsesItsCachedVersion(t *testing.T) { + f := newFixture(t) + existing := &v1beta2.DynamoGraphDeploymentCandidate{ObjectMeta: metav1.ObjectMeta{ + Namespace: testNamespace, + Name: cn("a"), + Labels: map[string]string{LabelRunUID: testRunUID, LabelCandidateID: "a"}, + }} + if err := f.c.Create(context.Background(), existing); err != nil { + t.Fatal(err) + } + var cached v1beta2.DynamoGraphDeploymentCandidate + if err := f.c.Get(context.Background(), types.NamespacedName{Namespace: testNamespace, Name: cn("a")}, &cached); err != nil { + t.Fatal(err) + } + f.publish(snapshotJSON(t, PhaseRunning, cand{id: "a"})) + f.mustReconcile() + if got := f.w.populateVersions[cn("a")]; got == "" || got != cached.ResourceVersion { + t.Fatalf("populate version = %q, want the cached version %q", got, cached.ResourceVersion) + } +} + +func TestConflictIsRetriedWithoutUsingUpTheFinalAttempts(t *testing.T) { + f := newFixture(t) + f.w.failOn["status"] = apierrors.NewConflict(schema.GroupResource{Group: "nvidia.com", Resource: "dynamographdeploymentruns"}, testRunName, errors.New("stale")) + f.w.sticky = true + f.publish(snapshotJSON(t, PhaseRunning, cand{id: "a"})) + f.exitSweeper(1) + for attempt := 0; attempt < 2*finalReconcileAttempts; attempt++ { + if err := f.reconcile(); !apierrors.IsConflict(err) || f.r.Completion.Finished() { + t.Fatalf("attempt %d: a conflict from a stale read must be retried, err=%v finished=%v", attempt, err, f.r.Completion.Finished()) + } + } + delete(f.w.failOn, "status") // the cache caught up + f.mustReconcile() + if !f.r.Completion.Finished() || f.r.Completion.Err() != nil { + t.Fatalf("finished=%v err=%v", f.r.Completion.Finished(), f.r.Completion.Err()) + } +} diff --git a/deploy/operator/internal/dgdrrunpublisher/writer.go b/deploy/operator/internal/dgdrrunpublisher/writer.go new file mode 100644 index 000000000000..a7f519e8306b --- /dev/null +++ b/deploy/operator/internal/dgdrrunpublisher/writer.go @@ -0,0 +1,235 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dgdrrunpublisher + +import ( + "context" + "encoding/json" + "time" + + "github.com/ai-dynamo/dynamo/deploy/operator/internal/dgdcreconcile" + v1beta2 "github.com/ai-dynamo/dynamo/deploy/operator/internal/dgdrrunpublisher/placeholderapi" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" +) + +// writer is every write the reconciler makes to the API server. It is a seam so the +// reconciler's behavior can be tested without server-side apply support in a fake client. +type writer interface { + // CreateCandidate creates the candidate and returns its resourceVersion as the API + // server has it. It must be idempotent. + CreateCandidate(ctx context.Context, run *v1beta2.DynamoGraphDeploymentRun, name string, candidate dgdcreconcile.DesiredCandidate) (resourceVersion string, err error) + // PopulateStatus records the candidate's evaluation facts and marks it Evaluated. The + // write is constrained to resourceVersion, so a stale view of the candidate fails + // with a conflict instead of rewriting status that was already populated. + PopulateStatus(ctx context.Context, name, resourceVersion string, candidate dgdcreconcile.DesiredCandidate) error + // UpdateRunStatus writes the publisher-owned run status fields, constrained to the + // resourceVersion of the run the decision to write was based on. + UpdateRunStatus(ctx context.Context, resourceVersion string, status RunStatus) error + // DeleteCandidate deletes the candidate with the given UID precondition. + DeleteCandidate(ctx context.Context, name string, uid types.UID) error +} + +// clientWriter writes with server-side apply through a controller-runtime client. +type clientWriter struct { + Client client.Client + // Reader reads straight from the API server. A candidate's resourceVersion right + // after its creation cannot come from the informer cache, which has not seen it yet. + Reader client.Reader + Namespace string + RunName string + RunUID string + // Now stamps the Evaluated condition; tests replace it. + Now func() time.Time +} + +var _ writer = (*clientWriter)(nil) + +func (w *clientWriter) apply(ctx context.Context, object *unstructured.Unstructured, status bool) error { + configuration := client.ApplyConfigurationFromUnstructured(object) + if status { + return w.Client.Status().Apply(ctx, configuration, client.FieldOwner(fieldOwner), client.ForceOwnership) + } + return w.Client.Apply(ctx, configuration, client.FieldOwner(fieldOwner), client.ForceOwnership) +} + +// CreateCandidate applies the candidate. It is only called for a candidate missing from +// the cache, and re-applying the same content after a stale read is a no-op, so no spec +// comparison is needed. +func (w *clientWriter) CreateCandidate(ctx context.Context, run *v1beta2.DynamoGraphDeploymentRun, name string, candidate dgdcreconcile.DesiredCandidate) (string, error) { + object, err := buildCandidateApply(w.Client.Scheme(), w.Namespace, w.RunUID, run, name, candidate) + if err != nil { + return "", err + } + if err := w.apply(ctx, object, false); err != nil { + return "", err + } + var stored v1beta2.DynamoGraphDeploymentCandidate + if err := w.Reader.Get(ctx, types.NamespacedName{Namespace: w.Namespace, Name: name}, &stored); err != nil { + return "", err + } + return stored.ResourceVersion, nil +} + +func (w *clientWriter) PopulateStatus(ctx context.Context, name, resourceVersion string, candidate dgdcreconcile.DesiredCandidate) error { + now := time.Now + if w.Now != nil { + now = w.Now + } + object, err := buildCandidateStatusApply(w.Namespace, name, resourceVersion, candidate, now()) + if err != nil { + return err + } + return w.apply(ctx, object, true) +} + +func (w *clientWriter) UpdateRunStatus(ctx context.Context, resourceVersion string, status RunStatus) error { + return w.apply(ctx, buildRunStatusApply(w.Namespace, w.RunName, resourceVersion, status), true) +} + +func (w *clientWriter) DeleteCandidate(ctx context.Context, name string, uid types.UID) error { + object := &v1beta2.DynamoGraphDeploymentCandidate{ObjectMeta: metav1.ObjectMeta{Namespace: w.Namespace, Name: name}} + var options []client.DeleteOption + if uid != "" { + options = append(options, client.Preconditions{UID: &uid}) + } + return client.IgnoreNotFound(w.Client.Delete(ctx, object, options...)) +} + +// buildCandidateApply is the server-side-apply payload that creates a candidate: labels +// and controller owner reference, the DGD spec with the resolved parameters, and the +// companion resources as an annotation. The status goes through the status subresource. +func buildCandidateApply(scheme *runtime.Scheme, namespace, runUID string, run *v1beta2.DynamoGraphDeploymentRun, name string, candidate dgdcreconcile.DesiredCandidate) (*unstructured.Unstructured, error) { + spec, companions, err := decodeManifest(candidate) + if err != nil { + return nil, err + } + parameters, err := normalizeJSON(candidate.Parameters) + if err != nil { + return nil, err + } + object := &v1beta2.DynamoGraphDeploymentCandidate{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace, + Name: name, + Labels: map[string]string{LabelRunUID: runUID, LabelCandidateID: candidate.ID}, + }, + Spec: v1beta2.DynamoGraphDeploymentCandidateSpec{DynamoGraphDeploymentSpec: *spec}, + } + if companions != "" { + object.Annotations = map[string]string{AnnotationAdditionalResources: companions} + } + if err := controllerutil.SetControllerReference(run, object, scheme); err != nil { + return nil, err + } + content, err := runtime.DefaultUnstructuredConverter.ToUnstructured(object) + if err != nil { + return nil, err + } + delete(content, "status") + unstructured.RemoveNestedField(content, "metadata", "creationTimestamp") + if err := unstructured.SetNestedField(content, parameters, "spec", "parameters"); err != nil { + return nil, err + } + apply := &unstructured.Unstructured{Object: content} + apply.SetGroupVersionKind(v1beta2.GroupVersion.WithKind("DynamoGraphDeploymentCandidate")) + return apply, nil +} + +// withResourceVersion constrains a server-side apply to one version of the object: the +// API server rejects it with a conflict if the object has changed since. +func withResourceVersion(metadata map[string]any, resourceVersion string) map[string]any { + if resourceVersion != "" { + metadata["resourceVersion"] = resourceVersion + } + return metadata +} + +// buildCandidateStatusApply is the status payload of a candidate: the metrics and the +// Evaluated condition. Candidate status is written once and then stays immutable, so the +// payload carries the resourceVersion the "not populated yet" decision was based on. +func buildCandidateStatusApply(namespace, name, resourceVersion string, candidate dgdcreconcile.DesiredCandidate, now time.Time) (*unstructured.Unstructured, error) { + metrics, err := normalizeJSON(candidate.Metrics) + if err != nil { + return nil, err + } + apply := &unstructured.Unstructured{Object: map[string]any{ + "metadata": withResourceVersion(map[string]any{"namespace": namespace, "name": name}, resourceVersion), + "status": map[string]any{ + "metrics": metrics, + "conditions": []any{map[string]any{ + "type": evaluatedCondition, + "status": string(metav1.ConditionTrue), + "reason": evaluatedCondition, + "message": "evaluated by the Sweeper", + "lastTransitionTime": now.UTC().Format(time.RFC3339), + }}, + }, + }} + apply.SetGroupVersionKind(v1beta2.GroupVersion.WithKind("DynamoGraphDeploymentCandidate")) + return apply, nil +} + +// buildRunStatusApply is the status payload of the run. It lists every publisher-owned +// field it wants to keep: under server-side apply a field this manager owns and omits +// from the payload is removed. It carries the resourceVersion of the run the decision to +// write was based on, so a decision made from a stale cached run conflicts instead of +// overwriting a newer status. +func buildRunStatusApply(namespace, runName, resourceVersion string, status RunStatus) *unstructured.Unstructured { + refs := make([]any, 0, len(status.CandidateNames)) + for _, name := range status.CandidateNames { + refs = append(refs, map[string]any{"name": name}) + } + fields := map[string]any{ + "message": status.Message, + "candidateRefs": refs, + } + if status.HasProgress { + fields["progress"] = map[string]any{"round": int64(status.Round), "evaluated": int64(status.Evaluated)} + } + if !status.LastProgress.IsZero() { + fields["lastProgressTime"] = status.LastProgress.UTC().Format(time.RFC3339) + } + apply := &unstructured.Unstructured{Object: map[string]any{ + "metadata": withResourceVersion(map[string]any{"namespace": namespace, "name": runName}, resourceVersion), + "status": fields, + }} + apply.SetGroupVersionKind(v1beta2.GroupVersion.WithKind("DynamoGraphDeploymentRun")) + return apply +} + +// normalizeJSON round-trips a value through JSON so it only holds the types unstructured +// objects allow (int64, float64, string, bool, maps, slices). +func normalizeJSON(value map[string]any) (map[string]any, error) { + out := map[string]any{} + if value == nil { + return out, nil + } + raw, err := json.Marshal(value) + if err != nil { + return nil, err + } + if err := json.Unmarshal(raw, &out); err != nil { + return nil, err + } + return out, nil +} From b1698bd738e21ab62fbd978cc8fddc3b2c984e35 Mon Sep 17 00:00:00 2001 From: devivasudevan <49675305+devivasudevan@users.noreply.github.com> Date: Tue, 13 Oct 2026 14:45:08 +0200 Subject: [PATCH 15/40] test(aisimulate): drop redundant inline comments in dgdr_run adapter test --- .../src/dynamo/aisimulate/tests/dgdr_run/test_adapter.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/src/dynamo/aisimulate/tests/dgdr_run/test_adapter.py b/components/src/dynamo/aisimulate/tests/dgdr_run/test_adapter.py index 75cc68cb5d38..eb36b89b11b7 100644 --- a/components/src/dynamo/aisimulate/tests/dgdr_run/test_adapter.py +++ b/components/src/dynamo/aisimulate/tests/dgdr_run/test_adapter.py @@ -291,12 +291,12 @@ def test_many_callbacks_in_one_interval_produce_one_periodic_write( config = make_config(tmp_path, snapshot_interval_seconds=0.5) adapter = DGDRRunOutputAdapter(config, workload=None) adapter.start() - assert adapter.snapshot_writes == 1 # initial + assert adapter.snapshot_writes == 1 for index in range(300): adapter.on_candidate(record(f"p{index:03d}", score=float(index))) time.sleep(0.15) - assert adapter.snapshot_writes == 1 # still inside the interval + assert adapter.snapshot_writes == 1 wait_for(lambda: adapter.snapshot_writes == 2, timeout=5) snapshot = load(tmp_path) From 4f927a31b769a9e9c7694b92823153f0bf4c26e6 Mon Sep 17 00:00:00 2001 From: devivasudevan <49675305+devivasudevan@users.noreply.github.com> Date: Wed, 14 Oct 2026 11:26:28 +0200 Subject: [PATCH 16/40] fix(dgdr-run-publisher): drop the reconciler's write history; decide from observed state Remove written/applied/lastProgress from the Reconciler. Each pass decides from the cached run and candidates; status writes stay constrained to the observed resourceVersion. CreateCandidate returns the server's copy so a stale cache cannot make the publisher rewrite populated candidate status. --- .../operator/internal/dgdrrunpublisher/doc.go | 8 +- .../internal/dgdrrunpublisher/reconciler.go | 66 ++++------- .../dgdrrunpublisher/reconciler_test.go | 112 +++++++++++++----- .../internal/dgdrrunpublisher/writer.go | 38 ++++-- 4 files changed, 138 insertions(+), 86 deletions(-) diff --git a/deploy/operator/internal/dgdrrunpublisher/doc.go b/deploy/operator/internal/dgdrrunpublisher/doc.go index 3d5a83cbee45..6fa0ed9ed68b 100644 --- a/deploy/operator/internal/dgdrrunpublisher/doc.go +++ b/deploy/operator/internal/dgdrrunpublisher/doc.go @@ -34,9 +34,11 @@ // where they differ, and finally deletes obsolete candidates. Failed writes are // retried by the workqueue with backoff. // -// The publisher's own writes come back as watch events and the cache can trail them, so -// the Reconciler remembers what it has written and ignores candidate update events; see -// Reconciler. +// The publisher's own writes come back as watch events and the cache can trail them. The +// Reconciler keeps no record of what it wrote: it decides from the current observations, +// constrains its status writes to the resourceVersion it observed (a stale decision +// conflicts and is retried through the workqueue), and ignores candidate update events; +// see Reconciler. // // The manager's cache covers only this Run, its candidates (by the run-UID label) and the // publisher's own Pod, so there is no API-server polling and no spec comparison: diff --git a/deploy/operator/internal/dgdrrunpublisher/reconciler.go b/deploy/operator/internal/dgdrrunpublisher/reconciler.go index 68c916f1970a..d3dd6d30b154 100644 --- a/deploy/operator/internal/dgdrrunpublisher/reconciler.go +++ b/deploy/operator/internal/dgdrrunpublisher/reconciler.go @@ -22,7 +22,6 @@ import ( "errors" "fmt" "strings" - "time" dgdv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1" v1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" @@ -73,17 +72,20 @@ const ( // Job pod names cannot be listed in resourceNames). // // Write echoes and cache lag: the publisher's own writes come back as watch events, and -// the cache can trail them. The reconciler therefore remembers what it has written (the -// candidates it created and populated, the last run status it applied, the latest -// progress time) and does not redo that work while the cache catches up. A retry after a -// partial success repeats only the unfinished writes, and an event that merely echoes a -// write finds nothing to do and produces no further write, so it cannot start a loop. +// the cache can trail them. The reconciler keeps no record of its writes. Every pass +// decides from the current observations (the cached run and candidates, the latest input +// snapshot), so a repeated pass while the cache catches up may repeat an idempotent +// write, which is harmless, and an echo of a write finds nothing left to do. Anything +// that changes behind the publisher's back (a deleted candidate, a modified run status) +// is repaired on the next pass. // // Stale reads: every status write is constrained to the resourceVersion the decision was // based on (the cached run, the cached candidate, or the version read back from the API -// server for a candidate created a moment ago). A decision made from an older cache +// server for a candidate applied a moment ago). A decision made from an older cache // observation fails with a conflict instead of overwriting newer state, and is retried -// once the informer has delivered the newer version. +// through the workqueue once the informer has delivered the newer version. A candidate's +// status is only populated when the API server's own copy, read right after the apply, +// has none yet, so a stale cache can never make the publisher rewrite populated status. type Reconciler struct { Client client.Client Writer writer @@ -102,12 +104,6 @@ type Reconciler struct { // finalFailures counts failed reconciles after the Sweeper exited. finalFailures int - // written holds the candidates this process has created and populated. - written map[string]struct{} - // applied is the last run status written successfully. - applied *RunStatus - // lastProgress is the latest progress time written. - lastProgress time.Time } func (r *Reconciler) key() types.NamespacedName { @@ -222,33 +218,31 @@ func (r *Reconciler) apply(ctx context.Context, run *v1beta2.DynamoGraphDeployme if err != nil { return err } - if r.written == nil { - r.written = map[string]struct{}{} - } - + // populated names the candidates whose status the API server already has; they are + // known from the live read that follows each apply, not from the cache. + populated := map[string]struct{}{} for _, candidate := range actions.Creates { name := desired.Names[candidate.ID] - if _, done := r.written[name]; done { - continue // created and populated by this process; the cache has not caught up yet - } - version, err := r.Writer.CreateCandidate(ctx, run, name, candidate) + stored, err := r.Writer.CreateCandidate(ctx, run, name, candidate) if err != nil { return fmt.Errorf("creating candidate %s: %w", candidate.ID, err) } - versions[name] = version // a new candidate is not in the cache; use the version the API server returned + versions[name] = stored.ResourceVersion // a new candidate is not in the cache; use the version the API server returned + if stored.Populated { + populated[name] = struct{}{} + } } unpopulated := make([]dgdcreconcile.DesiredCandidate, 0, len(actions.Creates)+len(actions.Incomplete)) unpopulated = append(unpopulated, actions.Creates...) unpopulated = append(unpopulated, actions.Incomplete...) for _, candidate := range unpopulated { name := desired.Names[candidate.ID] - if _, done := r.written[name]; done { + if _, done := populated[name]; done { continue } if err := r.Writer.PopulateStatus(ctx, name, versions[name], candidate); err != nil { return fmt.Errorf("populating status of candidate %s: %w", candidate.ID, err) } - r.written[name] = struct{}{} } if err := r.updateRunStatus(ctx, run, desired.Status); err != nil { @@ -263,30 +257,20 @@ func (r *Reconciler) apply(ctx context.Context, run *v1beta2.DynamoGraphDeployme return nil } -// updateRunStatus writes the run status when it differs from what the cache shows and -// from what this process already wrote. A snapshot without a usable timestamp keeps the -// latest progress time known, because a server-side-apply payload that omits the field -// would remove it. +// updateRunStatus writes the run status when it differs from what the cache shows. A +// snapshot without a usable timestamp keeps the progress time the cached run already has, +// because a server-side-apply payload that omits the field would remove it. The write is +// constrained to the cached run's resourceVersion, so a decision made from a stale cache +// fails with a conflict instead of overwriting newer status. func (r *Reconciler) updateRunStatus(ctx context.Context, run *v1beta2.DynamoGraphDeploymentRun, desired RunStatus) error { observed := observedStatus(run) if !desired.NeedsUpdate(observed) { return nil } - known := observed.LastProgress - if r.lastProgress.After(known) { - known = r.lastProgress - } - write := desired.WithPreservedProgress(known) - if r.applied != nil && !write.NeedsUpdate(*r.applied) { - return nil // already written; the cache has not caught up yet - } + write := desired.WithPreservedProgress(observed.LastProgress) if err := r.Writer.UpdateRunStatus(ctx, run.ResourceVersion, write); err != nil { return fmt.Errorf("updating run status: %w", err) } - r.applied = &write - if write.LastProgress.After(r.lastProgress) { - r.lastProgress = write.LastProgress - } return nil } diff --git a/deploy/operator/internal/dgdrrunpublisher/reconciler_test.go b/deploy/operator/internal/dgdrrunpublisher/reconciler_test.go index 8387962b2ff6..009bb309f6fd 100644 --- a/deploy/operator/internal/dgdrrunpublisher/reconciler_test.go +++ b/deploy/operator/internal/dgdrrunpublisher/reconciler_test.go @@ -50,8 +50,8 @@ const ( // memWriter is the reconciler's writer for tests. It logs every write attempt (a failed // attempt is logged with a trailing "!") and, unless lag is set, applies the write to // the fake client the reconciler reads from, which stands in for the informer cache. -// With lag set, writes are logged but never become visible, like a cache that has not -// caught up. +// With lag set, writes are logged but never become visible to the reconciler, like a +// cache that has not caught up; the server map still records them. type memWriter struct { t *testing.T c client.Client @@ -62,6 +62,9 @@ type memWriter struct { statusVersions []string populateVersions map[string]string lag bool + // server is what the API server holds, whatever the cache shows: the candidates that + // exist, and whether each has its status. + server map[string]bool // failOn makes the named operation ("create", "populate", "status", "delete") fail; // once, unless sticky. failOn map[string]error @@ -87,13 +90,16 @@ func (w *memWriter) record(op string, err error) error { return err } -func (w *memWriter) CreateCandidate(ctx context.Context, _ *v1beta2.DynamoGraphDeploymentRun, name string, candidate dgdcreconcile.DesiredCandidate) (string, error) { +func (w *memWriter) CreateCandidate(ctx context.Context, _ *v1beta2.DynamoGraphDeploymentRun, name string, candidate dgdcreconcile.DesiredCandidate) (StoredCandidate, error) { if err := w.record("create:"+name, w.fail("create")); err != nil { - return "", err + return StoredCandidate{}, err } - version := createdVersion(name) + // The apply is idempotent; the read-back reports what the server holds. + populated := w.server[name] + w.server[name] = populated + stored := StoredCandidate{ResourceVersion: createdVersion(name), Populated: populated} if w.lag { - return version, nil + return stored, nil } object := &v1beta2.DynamoGraphDeploymentCandidate{ObjectMeta: metav1.ObjectMeta{ Namespace: testNamespace, @@ -101,7 +107,7 @@ func (w *memWriter) CreateCandidate(ctx context.Context, _ *v1beta2.DynamoGraphD UID: types.UID("uid-" + name), Labels: map[string]string{LabelRunUID: testRunUID, LabelCandidateID: candidate.ID}, }} - return version, client.IgnoreAlreadyExists(w.c.Create(ctx, object)) + return stored, client.IgnoreAlreadyExists(w.c.Create(ctx, object)) } // createdVersion is the resourceVersion the fake API server "returns" for a new candidate. @@ -112,6 +118,7 @@ func (w *memWriter) PopulateStatus(ctx context.Context, name, resourceVersion st return err } w.populateVersions[name] = resourceVersion + w.server[name] = true if w.lag { return nil } @@ -182,7 +189,7 @@ func newFixture(t *testing.T) *fixture { MinInterval: time.Millisecond, MaxInterval: time.Millisecond, } - w := &memWriter{t: t, c: c, failOn: map[string]error{}, populateVersions: map[string]string{}} + w := &memWriter{t: t, c: c, failOn: map[string]error{}, populateVersions: map[string]string{}, server: map[string]bool{}} return &fixture{t: t, c: c, w: w, r: &Reconciler{ Client: c, Writer: w, @@ -230,6 +237,17 @@ func (f *fixture) exitSweeper(code int32) { } } +// deleteCandidate removes a candidate behind the publisher's back, from the API server +// and from the cache. +func (f *fixture) deleteCandidate(name string) { + f.t.Helper() + delete(f.w.server, name) + object := &v1beta2.DynamoGraphDeploymentCandidate{ObjectMeta: metav1.ObjectMeta{Namespace: testNamespace, Name: name}} + if err := f.c.Delete(context.Background(), object); err != nil { + f.t.Fatal(err) + } +} + func (f *fixture) takeOps() []string { ops := f.w.ops f.w.ops = nil @@ -299,25 +317,60 @@ func TestReconcilingUnchangedStateWritesNothing(t *testing.T) { } } -func TestCacheLagDoesNotRepeatWrites(t *testing.T) { +func TestCacheLagRepeatsOnlyIdempotentWrites(t *testing.T) { f := newFixture(t) f.w.lag = true // the cache never shows what was written f.publish(snapshotJSON(t, PhaseRunning, cand{id: "a"})) f.mustReconcile() - first := f.takeOps() - if len(first) != 3 { - t.Fatalf("first reconcile ops = %v", first) + if got, want := f.takeOps(), []string{"create:" + cn("a"), "populate:" + cn("a"), "status"}; !reflect.DeepEqual(got, want) { + t.Fatalf("first reconcile ops = %v, want %v", got, want) } f.mustReconcile() + // The candidate apply repeats harmlessly and the server reports the candidate populated, + // so its status is not written a second time. The run status is written again, still + // constrained to the cached version: against a real API server the first write has + // moved the run on, so this one is rejected as stale and retried through the workqueue. + if got, want := f.takeOps(), []string{"create:" + cn("a"), "status"}; !reflect.DeepEqual(got, want) { + t.Fatalf("second reconcile ops = %v, want %v", got, want) + } + if len(f.w.statusVersions) != 2 || f.w.statusVersions[0] != f.w.statusVersions[1] { + t.Fatalf("both status writes must carry the cached run version, got %v", f.w.statusVersions) + } +} + +func TestExternallyDeletedCandidateIsRecreated(t *testing.T) { + f := newFixture(t) + f.publish(snapshotJSON(t, PhaseRunning, cand{id: "a"})) + f.mustReconcile() + f.takeOps() + f.deleteCandidate(cn("a")) f.mustReconcile() - if got := f.takeOps(); len(got) != 0 { - t.Fatalf("an echo-triggered reconcile must not redo writes the cache has not shown yet, got %v", got) + if got, want := f.takeOps(), []string{"create:" + cn("a"), "populate:" + cn("a")}; !reflect.DeepEqual(got, want) { + t.Fatalf("a candidate deleted behind the publisher's back must be repaired, ops = %v, want %v", got, want) + } +} + +func TestExternallyModifiedRunStatusIsRepaired(t *testing.T) { + f := newFixture(t) + f.publish(snapshotJSON(t, PhaseRunning, cand{id: "a"})) + f.mustReconcile() + f.takeOps() + var run v1beta2.DynamoGraphDeploymentRun + if err := f.c.Get(context.Background(), types.NamespacedName{Namespace: testNamespace, Name: testRunName}, &run); err != nil { + t.Fatal(err) + } + run.Status.Message = "changed by someone else" + if err := f.c.Update(context.Background(), &run); err != nil { + t.Fatal(err) + } + f.mustReconcile() + if got, want := f.takeOps(), []string{"status"}; !reflect.DeepEqual(got, want) { + t.Fatalf("a run status modified behind the publisher's back must be repaired, ops = %v, want %v", got, want) } } func TestPartialSuccessRetryRepeatsOnlyTheUnfinishedWrites(t *testing.T) { f := newFixture(t) - f.w.lag = true f.w.failOn["status"] = errors.New("api unavailable") f.publish(snapshotJSON(t, PhaseRunning, cand{id: "a"})) if err := f.reconcile(); err == nil { @@ -393,22 +446,19 @@ func TestCandidateWithoutStatusIsCompletedNotRecreated(t *testing.T) { } func TestProgressTimeSurvivesASnapshotWithoutATimestamp(t *testing.T) { - for name, lag := range map[string]bool{"cache current": false, "cache lagging": true} { - t.Run(name, func(t *testing.T) { - f := newFixture(t) - f.w.lag = lag - first := time.Date(2026, 10, 5, 21, 35, 12, 0, time.UTC) - f.publish(withTimestamp(t, snapshotJSON(t, PhaseRunning, cand{id: "a"}), "2026-10-05T21:35:12Z")) - f.mustReconcile() - f.publish(withTimestamp(t, snapshotJSON(t, PhaseRunning, cand{id: "a"}, cand{id: "b"}), "")) - f.mustReconcile() - if len(f.w.status) != 2 { - t.Fatalf("want two status writes, got %d", len(f.w.status)) - } - if got := f.lastStatus().LastProgress; !got.Equal(first) { - t.Fatalf("the second write must repeat the known progress time %v, got %v", first, got) - } - }) + f := newFixture(t) + first := time.Date(2026, 10, 5, 21, 35, 12, 0, time.UTC) + f.publish(withTimestamp(t, snapshotJSON(t, PhaseRunning, cand{id: "a"}), "2026-10-05T21:35:12Z")) + f.mustReconcile() + f.publish(withTimestamp(t, snapshotJSON(t, PhaseRunning, cand{id: "a"}, cand{id: "b"}), "")) + f.mustReconcile() + if len(f.w.status) != 2 { + t.Fatalf("want two status writes, got %d", len(f.w.status)) + } + // The preserved time comes from the cached run, the current observed state; a stale + // cache would lose the version check instead of the timestamp. + if got := f.lastStatus().LastProgress; !got.Equal(first) { + t.Fatalf("the second write must repeat the observed progress time %v, got %v", first, got) } } diff --git a/deploy/operator/internal/dgdrrunpublisher/writer.go b/deploy/operator/internal/dgdrrunpublisher/writer.go index a7f519e8306b..f1b376c13235 100644 --- a/deploy/operator/internal/dgdrrunpublisher/writer.go +++ b/deploy/operator/internal/dgdrrunpublisher/writer.go @@ -24,6 +24,7 @@ import ( "github.com/ai-dynamo/dynamo/deploy/operator/internal/dgdcreconcile" v1beta2 "github.com/ai-dynamo/dynamo/deploy/operator/internal/dgdrrunpublisher/placeholderapi" + "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" @@ -32,12 +33,22 @@ import ( "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" ) +// StoredCandidate is the API server's view of a candidate right after it was applied. +type StoredCandidate struct { + // ResourceVersion is the version a status write for this candidate is constrained to. + ResourceVersion string + // Populated reports that the candidate already has its evaluation status, for example + // when a stale cache led the reconciler to apply a candidate that exists. + Populated bool +} + // writer is every write the reconciler makes to the API server. It is a seam so the // reconciler's behavior can be tested without server-side apply support in a fake client. type writer interface { - // CreateCandidate creates the candidate and returns its resourceVersion as the API - // server has it. It must be idempotent. - CreateCandidate(ctx context.Context, run *v1beta2.DynamoGraphDeploymentRun, name string, candidate dgdcreconcile.DesiredCandidate) (resourceVersion string, err error) + // CreateCandidate applies the candidate and returns the API server's own copy of it, + // read after the apply. It must be idempotent: applying a candidate that already + // exists changes nothing. + CreateCandidate(ctx context.Context, run *v1beta2.DynamoGraphDeploymentRun, name string, candidate dgdcreconcile.DesiredCandidate) (StoredCandidate, error) // PopulateStatus records the candidate's evaluation facts and marks it Evaluated. The // write is constrained to resourceVersion, so a stale view of the candidate fails // with a conflict instead of rewriting status that was already populated. @@ -72,22 +83,27 @@ func (w *clientWriter) apply(ctx context.Context, object *unstructured.Unstructu return w.Client.Apply(ctx, configuration, client.FieldOwner(fieldOwner), client.ForceOwnership) } -// CreateCandidate applies the candidate. It is only called for a candidate missing from -// the cache, and re-applying the same content after a stale read is a no-op, so no spec -// comparison is needed. -func (w *clientWriter) CreateCandidate(ctx context.Context, run *v1beta2.DynamoGraphDeploymentRun, name string, candidate dgdcreconcile.DesiredCandidate) (string, error) { +// CreateCandidate applies the candidate and reads it back from the API server. It is +// only called for a candidate missing from the cache, and re-applying the same content +// to a candidate that exists is a no-op, so no spec comparison is needed. The read-back +// tells the caller whether the status is populated already: the cache cannot, because it +// is exactly what may be stale. +func (w *clientWriter) CreateCandidate(ctx context.Context, run *v1beta2.DynamoGraphDeploymentRun, name string, candidate dgdcreconcile.DesiredCandidate) (StoredCandidate, error) { object, err := buildCandidateApply(w.Client.Scheme(), w.Namespace, w.RunUID, run, name, candidate) if err != nil { - return "", err + return StoredCandidate{}, err } if err := w.apply(ctx, object, false); err != nil { - return "", err + return StoredCandidate{}, err } var stored v1beta2.DynamoGraphDeploymentCandidate if err := w.Reader.Get(ctx, types.NamespacedName{Namespace: w.Namespace, Name: name}, &stored); err != nil { - return "", err + return StoredCandidate{}, err } - return stored.ResourceVersion, nil + return StoredCandidate{ + ResourceVersion: stored.ResourceVersion, + Populated: meta.IsStatusConditionTrue(stored.Status.Conditions, evaluatedCondition), + }, nil } func (w *clientWriter) PopulateStatus(ctx context.Context, name, resourceVersion string, candidate dgdcreconcile.DesiredCandidate) error { From 62174ea0788d9581186b0a2c137174adc38833b7 Mon Sep 17 00:00:00 2001 From: devivasudevan <49675305+devivasudevan@users.noreply.github.com> Date: Thu, 15 Oct 2026 11:00:35 +0200 Subject: [PATCH 17/40] refactor(operator): run publisher on the v1beta2 Run and Candidate types --- .../operator/internal/dgdrrunpublisher/doc.go | 4 +- .../internal/dgdrrunpublisher/manager.go | 2 +- .../dgdrrunpublisher/placeholderapi/types.go | 273 ------------------ .../internal/dgdrrunpublisher/plan.go | 57 ++-- .../internal/dgdrrunpublisher/plan_test.go | 21 +- .../internal/dgdrrunpublisher/reconciler.go | 18 +- .../dgdrrunpublisher/reconciler_test.go | 65 +++-- .../internal/dgdrrunpublisher/writer.go | 54 ++-- 8 files changed, 99 insertions(+), 395 deletions(-) delete mode 100644 deploy/operator/internal/dgdrrunpublisher/placeholderapi/types.go diff --git a/deploy/operator/internal/dgdrrunpublisher/doc.go b/deploy/operator/internal/dgdrrunpublisher/doc.go index 6fa0ed9ed68b..18b02a637102 100644 --- a/deploy/operator/internal/dgdrrunpublisher/doc.go +++ b/deploy/operator/internal/dgdrrunpublisher/doc.go @@ -30,8 +30,8 @@ // 2. The Reconciler consumes the key. Informer events for the Run, its candidates and // this Pod enqueue the same key. It reads the latest stored input and the cached // cluster state, then creates missing candidates with server-side apply, completes -// candidate status, updates the Run's progress, message and ordered references only -// where they differ, and finally deletes obsolete candidates. Failed writes are +// candidate status (its metrics), updates the Run's published-candidate count, progress +// time and ordered references only where they differ, and finally deletes obsolete candidates. Failed writes are // retried by the workqueue with backoff. // // The publisher's own writes come back as watch events and the cache can trail them. The diff --git a/deploy/operator/internal/dgdrrunpublisher/manager.go b/deploy/operator/internal/dgdrrunpublisher/manager.go index ffd38e7d20f5..8c67862a1684 100644 --- a/deploy/operator/internal/dgdrrunpublisher/manager.go +++ b/deploy/operator/internal/dgdrrunpublisher/manager.go @@ -25,7 +25,7 @@ import ( "time" v1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" - v1beta2 "github.com/ai-dynamo/dynamo/deploy/operator/internal/dgdrrunpublisher/placeholderapi" + v1beta2 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta2" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/labels" diff --git a/deploy/operator/internal/dgdrrunpublisher/placeholderapi/types.go b/deploy/operator/internal/dgdrrunpublisher/placeholderapi/types.go deleted file mode 100644 index a743ed40a615..000000000000 --- a/deploy/operator/internal/dgdrrunpublisher/placeholderapi/types.go +++ /dev/null @@ -1,273 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// PLACEHOLDER -- not the real v1beta2 API (and deliberately not under api/, so it is not a served -// or converted API version). This package exists only so the DGDR(v2) -// run publisher (internal/dgdrrunpublisher) has something to compile against before the -// real v1beta2 types exist on this branch. The real, reviewed types are PRs #13603 -// ("feat: add api for dgdr v1beta2") and #13744 ("feat: dgdr v1beta2 conversion"), -// still feature-gated/inactive when this file was written. The shapes below follow the -// review of #15554: a candidate is an immutable evaluated point (spec = the DGD fields -// plus the resolved evaluation parameters, status = metrics + evaluation conditions) -// and ordering lives on the run's status, never on the candidate. Delete this package when the real types land; -// only import paths change. -// -// Deliberately NOT included: conversion webhooks, the full Request type, and -// deepcopy-gen output (DeepCopyObject is hand-written here). -package placeholderapi - -import ( - v1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/schema" -) - -// GroupVersion is the group version used to register these placeholder objects. -var GroupVersion = schema.GroupVersion{Group: "nvidia.com", Version: "v1beta2"} - -// AddToScheme registers the placeholder types with the given scheme. -func AddToScheme(scheme *runtime.Scheme) error { - scheme.AddKnownTypes(GroupVersion, - &DynamoGraphDeploymentCandidate{}, - &DynamoGraphDeploymentCandidateList{}, - &DynamoGraphDeploymentRun{}, - &DynamoGraphDeploymentRunList{}, - ) - metav1.AddToGroupVersion(scheme, GroupVersion) - return nil -} - -// DynamoGraphDeploymentCandidateSpec is the immutable evaluated point: the flat DGD -// fields plus the resolved evaluation parameters. -type DynamoGraphDeploymentCandidateSpec struct { - v1beta1.DynamoGraphDeploymentSpec `json:",inline"` - - // Parameters are the resolved evaluation parameters of the point. - // +optional - Parameters *runtime.RawExtension `json:"parameters,omitempty"` -} - -// DynamoGraphDeploymentCandidateStatus records evaluation results. It is populated once -// after creation and never updated afterwards; it carries no rank. -type DynamoGraphDeploymentCandidateStatus struct { - // Metrics are the evaluation metrics of the point. - // +optional - Metrics *runtime.RawExtension `json:"metrics,omitempty"` - - // Conditions describes evaluation and materialization. - // +optional - Conditions []metav1.Condition `json:"conditions,omitempty"` -} - -// DynamoGraphDeploymentCandidate is one bounded, user-visible search result. -type DynamoGraphDeploymentCandidate struct { - metav1.TypeMeta `json:",inline"` - metav1.ObjectMeta `json:"metadata,omitempty"` - - Spec DynamoGraphDeploymentCandidateSpec `json:"spec"` - Status DynamoGraphDeploymentCandidateStatus `json:"status,omitempty"` -} - -type DynamoGraphDeploymentCandidateList struct { - metav1.TypeMeta `json:",inline"` - metav1.ListMeta `json:"metadata,omitempty"` - Items []DynamoGraphDeploymentCandidate `json:"items"` -} - -// CandidateRef is an ordered reference from a run to one of its candidates. -type CandidateRef struct { - Name string `json:"name"` -} - -// RunProgress is the Sweeper progress reported by the snapshot. -type RunProgress struct { - Round int32 `json:"round"` - Evaluated int32 `json:"evaluated"` -} - -// DynamoGraphDeploymentRunSpec is intentionally empty in this placeholder: the real -// spec is an alias of the request spec (see #13603). -type DynamoGraphDeploymentRunSpec struct{} - -// DynamoGraphDeploymentRunStatus holds everything that changes while a search runs. -// Rank is the order of CandidateRefs (best first for scalar searches). The terminal -// Completed condition is derived by the run controller from the Job result, not -// written by the publisher. -type DynamoGraphDeploymentRunStatus struct { - // Message is a human-readable summary of the latest state or failure. - // +optional - Message string `json:"message,omitempty"` - - // Progress is the latest Sweeper progress. - // +optional - Progress *RunProgress `json:"progress,omitempty"` - - // LastProgressTime is when the Sweeper last published new state. - // +optional - LastProgressTime *metav1.Time `json:"lastProgressTime,omitempty"` - - // CandidateRefs lists the run's candidates in rank order. - // +optional - CandidateRefs []CandidateRef `json:"candidateRefs,omitempty"` - - // Conditions describes the run. - // +optional - Conditions []metav1.Condition `json:"conditions,omitempty"` -} - -// DynamoGraphDeploymentRun is one execution of a search request. -type DynamoGraphDeploymentRun struct { - metav1.TypeMeta `json:",inline"` - metav1.ObjectMeta `json:"metadata,omitempty"` - - Spec DynamoGraphDeploymentRunSpec `json:"spec,omitempty"` - Status DynamoGraphDeploymentRunStatus `json:"status,omitempty"` -} - -type DynamoGraphDeploymentRunList struct { - metav1.TypeMeta `json:",inline"` - metav1.ListMeta `json:"metadata,omitempty"` - Items []DynamoGraphDeploymentRun `json:"items"` -} - -// -- hand-written runtime.Object plumbing (deepcopy-gen output in the real PR) -- - -func (in *DynamoGraphDeploymentCandidateSpec) DeepCopyInto(out *DynamoGraphDeploymentCandidateSpec) { - *out = *in - in.DynamoGraphDeploymentSpec.DeepCopyInto(&out.DynamoGraphDeploymentSpec) - if in.Parameters != nil { - out.Parameters = in.Parameters.DeepCopy() - } -} - -func (in *DynamoGraphDeploymentCandidateStatus) DeepCopyInto(out *DynamoGraphDeploymentCandidateStatus) { - *out = *in - if in.Metrics != nil { - out.Metrics = in.Metrics.DeepCopy() - } - if in.Conditions != nil { - out.Conditions = make([]metav1.Condition, len(in.Conditions)) - for i := range in.Conditions { - in.Conditions[i].DeepCopyInto(&out.Conditions[i]) - } - } -} - -func (in *DynamoGraphDeploymentCandidate) DeepCopyInto(out *DynamoGraphDeploymentCandidate) { - *out = *in - out.TypeMeta = in.TypeMeta - in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) - in.Spec.DeepCopyInto(&out.Spec) - in.Status.DeepCopyInto(&out.Status) -} - -func (in *DynamoGraphDeploymentCandidate) DeepCopy() *DynamoGraphDeploymentCandidate { - if in == nil { - return nil - } - out := new(DynamoGraphDeploymentCandidate) - in.DeepCopyInto(out) - return out -} - -func (in *DynamoGraphDeploymentCandidate) DeepCopyObject() runtime.Object { return in.DeepCopy() } - -func (in *DynamoGraphDeploymentCandidateList) DeepCopyInto(out *DynamoGraphDeploymentCandidateList) { - *out = *in - out.TypeMeta = in.TypeMeta - out.ListMeta = in.ListMeta - if in.Items != nil { - out.Items = make([]DynamoGraphDeploymentCandidate, len(in.Items)) - for i := range in.Items { - in.Items[i].DeepCopyInto(&out.Items[i]) - } - } -} - -func (in *DynamoGraphDeploymentCandidateList) DeepCopy() *DynamoGraphDeploymentCandidateList { - if in == nil { - return nil - } - out := new(DynamoGraphDeploymentCandidateList) - in.DeepCopyInto(out) - return out -} - -func (in *DynamoGraphDeploymentCandidateList) DeepCopyObject() runtime.Object { return in.DeepCopy() } - -func (in *DynamoGraphDeploymentRunStatus) DeepCopyInto(out *DynamoGraphDeploymentRunStatus) { - *out = *in - if in.Progress != nil { - progress := *in.Progress - out.Progress = &progress - } - if in.LastProgressTime != nil { - out.LastProgressTime = in.LastProgressTime.DeepCopy() - } - if in.CandidateRefs != nil { - out.CandidateRefs = append([]CandidateRef(nil), in.CandidateRefs...) - } - if in.Conditions != nil { - out.Conditions = make([]metav1.Condition, len(in.Conditions)) - for i := range in.Conditions { - in.Conditions[i].DeepCopyInto(&out.Conditions[i]) - } - } -} - -func (in *DynamoGraphDeploymentRun) DeepCopyInto(out *DynamoGraphDeploymentRun) { - *out = *in - out.TypeMeta = in.TypeMeta - in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) - out.Spec = in.Spec - in.Status.DeepCopyInto(&out.Status) -} - -func (in *DynamoGraphDeploymentRun) DeepCopy() *DynamoGraphDeploymentRun { - if in == nil { - return nil - } - out := new(DynamoGraphDeploymentRun) - in.DeepCopyInto(out) - return out -} - -func (in *DynamoGraphDeploymentRun) DeepCopyObject() runtime.Object { return in.DeepCopy() } - -func (in *DynamoGraphDeploymentRunList) DeepCopyInto(out *DynamoGraphDeploymentRunList) { - *out = *in - out.TypeMeta = in.TypeMeta - out.ListMeta = in.ListMeta - if in.Items != nil { - out.Items = make([]DynamoGraphDeploymentRun, len(in.Items)) - for i := range in.Items { - in.Items[i].DeepCopyInto(&out.Items[i]) - } - } -} - -func (in *DynamoGraphDeploymentRunList) DeepCopy() *DynamoGraphDeploymentRunList { - if in == nil { - return nil - } - out := new(DynamoGraphDeploymentRunList) - in.DeepCopyInto(out) - return out -} - -func (in *DynamoGraphDeploymentRunList) DeepCopyObject() runtime.Object { return in.DeepCopy() } diff --git a/deploy/operator/internal/dgdrrunpublisher/plan.go b/deploy/operator/internal/dgdrrunpublisher/plan.go index ec2bc2ee4750..3f1efa3601f4 100644 --- a/deploy/operator/internal/dgdrrunpublisher/plan.go +++ b/deploy/operator/internal/dgdrrunpublisher/plan.go @@ -21,7 +21,6 @@ import ( "crypto/sha256" "encoding/hex" "slices" - "strconv" "strings" "time" @@ -35,14 +34,19 @@ const SnapshotFileName = "dgdr_run_snapshot.yaml" const maxObjectName = 253 // RunStatus is the part of DynamoGraphDeploymentRun.status the publisher owns: the -// ordered candidate references plus progress. CandidateNames is in rank order (best -// first for scalar searches). LastProgress is the zero time when the snapshot carries -// no usable timestamp, in which case the existing value is left untouched. +// ordered candidate references, the number of published candidates +// (status.progress.candidates.published) and the coarse progress time. CandidateNames is +// in rank order (best first for scalar searches). Published always equals +// len(CandidateNames) in a desired status; it is kept separately so a status whose +// progress block was removed or edited by someone else is noticed and repaired. +// LastProgress is the zero time when the snapshot carries no usable timestamp, in which +// case the existing value is left untouched. +// +// The snapshot's round and evaluated counters have no counterpart in the Run status: it +// reports per-branch rounds and outcome counters that the snapshot does not carry yet, +// so they are not published rather than guessed. type RunStatus struct { - Message string - HasProgress bool - Round int32 - Evaluated int32 + Published int32 LastProgress time.Time CandidateNames []string } @@ -56,6 +60,9 @@ type DesiredState struct { Status RunStatus Terminal bool Phase string + // Failed counts the candidates whose materialization failed. They are not created + // and the Run status has no field for them, so they are only logged. + Failed int } // CandidateName is the DGDC name for a candidate id. It depends only on the run @@ -77,8 +84,8 @@ func CandidateName(runName, runUID, id string) string { } // Desire projects a snapshot onto the desired cluster state of one run incarnation. -// Candidates whose materialization failed are not created; they are only counted in the -// run message. +// Candidates whose materialization failed are not created; they are only counted in +// DesiredState.Failed. func Desire(runName, runUID string, snap *Snapshot) DesiredState { state := DesiredState{ Names: make(map[string]string, len(snap.Candidates)), @@ -102,36 +109,15 @@ func Desire(runName, runUID string, snap *Snapshot) DesiredState { Metrics: candidate.Metrics, }) } + state.Failed = failed state.Status = RunStatus{ - Message: runMessage(snap, failed), - HasProgress: true, - Round: snap.Progress.Round, - Evaluated: snap.Progress.Evaluated, + Published: int32(len(names)), LastProgress: progressTime(snap), CandidateNames: names, } return state } -func runMessage(snap *Snapshot, failed int) string { - message := snap.Run.Message - if snap.Run.Error != "" { - message = snap.Run.Error - } - if failed > 0 { - suffix := fmtFailed(failed) - if message == "" { - return suffix - } - return message + "; " + suffix - } - return message -} - -func fmtFailed(failed int) string { - return strconv.Itoa(failed) + " candidate(s) failed materialization" -} - // progressTime is the snapshot's own timestamp (written by the Sweeper container in the // same pod) truncated to the second the API stores, or the zero time when it is missing // or malformed. Using the snapshot's time instead of the clock keeps reconciliation @@ -159,10 +145,7 @@ func (desired RunStatus) WithPreservedProgress(known time.Time) RunStatus { // the status is only written when something changed. A zero desired LastProgress means // "leave the observed value alone". func (desired RunStatus) NeedsUpdate(observed RunStatus) bool { - if desired.Message != observed.Message || desired.HasProgress != observed.HasProgress { - return true - } - if desired.HasProgress && (desired.Round != observed.Round || desired.Evaluated != observed.Evaluated) { + if desired.Published != observed.Published { return true } if !desired.LastProgress.IsZero() && !desired.LastProgress.Equal(observed.LastProgress) { diff --git a/deploy/operator/internal/dgdrrunpublisher/plan_test.go b/deploy/operator/internal/dgdrrunpublisher/plan_test.go index f9c24b01befd..c45ba1c6ea78 100644 --- a/deploy/operator/internal/dgdrrunpublisher/plan_test.go +++ b/deploy/operator/internal/dgdrrunpublisher/plan_test.go @@ -64,16 +64,18 @@ func TestFailedMaterializationIsNotDesiredButCounted(t *testing.T) { if !reflect.DeepEqual(state.Status.CandidateNames, []string{cn("a")}) { t.Fatalf("refs must not include the failed candidate: %v", state.Status.CandidateNames) } - if !strings.Contains(state.Status.Message, "1 candidate(s) failed materialization") { - t.Fatalf("message = %q", state.Status.Message) + if state.Failed != 1 { + t.Fatalf("failed = %d, want 1", state.Failed) + } + if state.Status.Published != 1 { + t.Fatalf("published = %d, want 1: failed candidates are not published", state.Status.Published) } } -func TestRunErrorReplacesTheMessage(t *testing.T) { - snap := mustParse(t, snapshotJSON(t, PhaseFailed, cand{id: "a"})) - snap.Run.Message, snap.Run.Error = "searching", "out of memory" - if got := Desire("run", testRunUID, snap).Status.Message; got != "out of memory" { - t.Fatalf("message = %q", got) +func TestPublishedCountsTheReferencedCandidates(t *testing.T) { + state := desire(t, snapshotJSON(t, PhaseRunning, cand{id: "a"}, cand{id: "b"})) + if got, want := state.Status.Published, int32(len(state.Status.CandidateNames)); got != want || got != 2 { + t.Fatalf("published = %d, refs = %d, want 2", got, want) } } @@ -105,10 +107,7 @@ func TestNeedsUpdateOnlyWhenSomethingDiffers(t *testing.T) { t.Fatal("identical status needs no write") } changes := map[string]func(*RunStatus){ - "message": func(s *RunStatus) { s.Message = "other" }, - "round": func(s *RunStatus) { s.Round++ }, - "evaluated": func(s *RunStatus) { s.Evaluated++ }, - "no progress": func(s *RunStatus) { s.HasProgress = false }, + "published": func(s *RunStatus) { s.Published++ }, "progress time": func(s *RunStatus) { s.LastProgress = s.LastProgress.Add(time.Second) }, "references": func(s *RunStatus) { s.CandidateNames = nil }, } diff --git a/deploy/operator/internal/dgdrrunpublisher/reconciler.go b/deploy/operator/internal/dgdrrunpublisher/reconciler.go index d3dd6d30b154..85cd6ad4efa8 100644 --- a/deploy/operator/internal/dgdrrunpublisher/reconciler.go +++ b/deploy/operator/internal/dgdrrunpublisher/reconciler.go @@ -25,11 +25,10 @@ import ( dgdv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1" v1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" + v1beta2 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta2" "github.com/ai-dynamo/dynamo/deploy/operator/internal/dgdcreconcile" - v1beta2 "github.com/ai-dynamo/dynamo/deploy/operator/internal/dgdrrunpublisher/placeholderapi" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" @@ -46,8 +45,6 @@ const ( // LabelCandidateID records the stable candidate id (the identity read back on list). LabelCandidateID = "nvidia.com/dgdr-candidate-id" - evaluatedCondition = "Evaluated" - // AnnotationAdditionalResources carries the companion resources (for example // generated ConfigMaps) rendered next to the DGD, as multi-document YAML. It is the // same key and format the DGDR controller uses. @@ -197,6 +194,9 @@ func (r *Reconciler) retryOrFail(exited bool, err error) error { // the status of candidates that lack it, update the run, then delete obsolete candidates. // It is idempotent and safe to retry; existing candidate specs are never touched. func (r *Reconciler) apply(ctx context.Context, run *v1beta2.DynamoGraphDeploymentRun, desired DesiredState) error { + if desired.Failed > 0 { + ctrl.LoggerFrom(ctx).V(1).Info("candidates failed materialization and are not published", "count", desired.Failed) + } var list v1beta2.DynamoGraphDeploymentCandidateList if err := r.Client.List(ctx, &list, client.InNamespace(r.Namespace), client.MatchingLabels{LabelRunUID: r.RunUID}); err != nil { return fmt.Errorf("listing candidates: %w", err) @@ -209,7 +209,7 @@ func (r *Reconciler) apply(ctx context.Context, run *v1beta2.DynamoGraphDeployme current = append(current, dgdcreconcile.CurrentDGDC{ Name: item.Name, ID: item.Labels[LabelCandidateID], - Incomplete: !meta.IsStatusConditionTrue(item.Status.Conditions, evaluatedCondition), + Incomplete: item.Status.Metrics == nil, }) uids[item.Name] = item.UID versions[item.Name] = item.ResourceVersion @@ -276,11 +276,9 @@ func (r *Reconciler) updateRunStatus(ctx context.Context, run *v1beta2.DynamoGra // observedStatus is the publisher-owned part of the run's current (cached) status. func observedStatus(run *v1beta2.DynamoGraphDeploymentRun) RunStatus { - observed := RunStatus{Message: run.Status.Message} - if progress := run.Status.Progress; progress != nil { - observed.HasProgress = true - observed.Round = progress.Round - observed.Evaluated = progress.Evaluated + var observed RunStatus + if progress := run.Status.Progress; progress != nil && progress.Candidates != nil { + observed.Published = progress.Candidates.Published } if run.Status.LastProgressTime != nil { observed.LastProgress = run.Status.LastProgressTime.UTC() diff --git a/deploy/operator/internal/dgdrrunpublisher/reconciler_test.go b/deploy/operator/internal/dgdrrunpublisher/reconciler_test.go index 009bb309f6fd..27ff47cc988d 100644 --- a/deploy/operator/internal/dgdrrunpublisher/reconciler_test.go +++ b/deploy/operator/internal/dgdrrunpublisher/reconciler_test.go @@ -27,13 +27,13 @@ import ( "testing" "time" + v1beta2 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta2" "github.com/ai-dynamo/dynamo/deploy/operator/internal/dgdcreconcile" - v1beta2 "github.com/ai-dynamo/dynamo/deploy/operator/internal/dgdrrunpublisher/placeholderapi" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" ctrl "sigs.k8s.io/controller-runtime" @@ -126,7 +126,7 @@ func (w *memWriter) PopulateStatus(ctx context.Context, name, resourceVersion st if err := w.c.Get(ctx, types.NamespacedName{Namespace: testNamespace, Name: name}, &object); err != nil { return err } - meta.SetStatusCondition(&object.Status.Conditions, metav1.Condition{Type: evaluatedCondition, Status: metav1.ConditionTrue, Reason: evaluatedCondition}) + object.Status.Metrics = &runtime.RawExtension{Raw: []byte("{}")} return w.c.Update(ctx, &object) } @@ -143,15 +143,14 @@ func (w *memWriter) UpdateRunStatus(ctx context.Context, resourceVersion string, if err := w.c.Get(ctx, types.NamespacedName{Namespace: testNamespace, Name: testRunName}, &run); err != nil { return err } - run.Status.Message = status.Message - run.Status.Progress = &v1beta2.RunProgress{Round: status.Round, Evaluated: status.Evaluated} + run.Status.Progress = &v1beta2.DGDRRunProgress{Candidates: &v1beta2.CandidateProgress{Published: status.Published}} if !status.LastProgress.IsZero() { progressed := metav1.NewTime(status.LastProgress) run.Status.LastProgressTime = &progressed } run.Status.CandidateRefs = nil for _, name := range status.CandidateNames { - run.Status.CandidateRefs = append(run.Status.CandidateRefs, v1beta2.CandidateRef{Name: name}) + run.Status.CandidateRefs = append(run.Status.CandidateRefs, corev1.LocalObjectReference{Name: name}) } return w.c.Update(ctx, &run) } @@ -359,7 +358,7 @@ func TestExternallyModifiedRunStatusIsRepaired(t *testing.T) { if err := f.c.Get(context.Background(), types.NamespacedName{Namespace: testNamespace, Name: testRunName}, &run); err != nil { t.Fatal(err) } - run.Status.Message = "changed by someone else" + run.Status.CandidateRefs = []corev1.LocalObjectReference{{Name: "changed-by-someone-else"}} if err := f.c.Update(context.Background(), &run); err != nil { t.Fatal(err) } @@ -423,7 +422,7 @@ func TestFailedMaterializationIsNotCreatedButCounted(t *testing.T) { } } status := f.lastStatus() - if !reflect.DeepEqual(status.CandidateNames, []string{cn("a")}) || !strings.Contains(status.Message, "1 candidate(s) failed materialization") { + if !reflect.DeepEqual(status.CandidateNames, []string{cn("a")}) || status.Published != 1 { t.Fatalf("status = %+v", status) } } @@ -679,16 +678,15 @@ func TestSweeperStateOfReadsOnlyTheSweeperContainer(t *testing.T) { func TestObservedStatusMirrorsTheRunStatus(t *testing.T) { when := metav1.NewTime(time.Date(2026, 10, 5, 21, 35, 12, 0, time.UTC)) run := &v1beta2.DynamoGraphDeploymentRun{Status: v1beta2.DynamoGraphDeploymentRunStatus{ - Message: "m", - Progress: &v1beta2.RunProgress{Round: 2, Evaluated: 5}, + Progress: &v1beta2.DGDRRunProgress{Candidates: &v1beta2.CandidateProgress{Published: 2}}, LastProgressTime: &when, - CandidateRefs: []v1beta2.CandidateRef{{Name: "x"}, {Name: "y"}}, + CandidateRefs: []corev1.LocalObjectReference{{Name: "x"}, {Name: "y"}}, }} - want := RunStatus{Message: "m", HasProgress: true, Round: 2, Evaluated: 5, LastProgress: when.Time, CandidateNames: []string{"x", "y"}} + want := RunStatus{Published: 2, LastProgress: when.Time, CandidateNames: []string{"x", "y"}} if got := observedStatus(run); !reflect.DeepEqual(got, want) { t.Fatalf("got %+v, want %+v", got, want) } - if got := observedStatus(&v1beta2.DynamoGraphDeploymentRun{}); got.HasProgress || !got.LastProgress.IsZero() { + if got := observedStatus(&v1beta2.DynamoGraphDeploymentRun{}); got.Published != 0 || !got.LastProgress.IsZero() { t.Fatalf("an empty status must not look like reported progress: %+v", got) } } @@ -765,12 +763,11 @@ func TestCandidateApplyPayload(t *testing.T) { func TestCandidateStatusApplyPayloadIsDeterministic(t *testing.T) { candidate := dgdcreconcile.DesiredCandidate{ID: "a", Metrics: map[string]any{"score": 2}} - now := time.Date(2026, 10, 5, 21, 35, 12, 0, time.UTC) - first, err := buildCandidateStatusApply(testNamespace, cn("a"), "17", candidate, now) + first, err := buildCandidateStatusApply(testNamespace, cn("a"), "17", candidate) if err != nil { t.Fatal(err) } - second, err := buildCandidateStatusApply(testNamespace, cn("a"), "17", candidate, now) + second, err := buildCandidateStatusApply(testNamespace, cn("a"), "17", candidate) if err != nil { t.Fatal(err) } @@ -780,20 +777,31 @@ func TestCandidateStatusApplyPayloadIsDeterministic(t *testing.T) { if got, _ := nested(t, first, "metadata", "resourceVersion"); got != "17" { t.Fatalf("the status write must be constrained to the version it was decided on, got %v", got) } - conditions, _ := nested(t, first, "status", "conditions") - list, ok := conditions.([]any) - if !ok || len(list) != 1 { - t.Fatalf("conditions = %v", conditions) + if _, found := nested(t, first, "status", "metrics"); !found { + t.Fatal("the metrics are the candidate's status") } - condition := list[0].(map[string]any) - if condition["type"] != evaluatedCondition || condition["status"] != "True" || condition["lastTransitionTime"] != "2026-10-05T21:35:12Z" { - t.Fatalf("condition = %v", condition) + if _, found := nested(t, first, "status", "conditions"); found { + t.Fatal("candidate status is metrics only") + } +} + +func TestCandidateStatusApplyAlwaysWritesMetricsEvenWhenEmpty(t *testing.T) { + apply, err := buildCandidateStatusApply(testNamespace, cn("a"), "17", dgdcreconcile.DesiredCandidate{ID: "a"}) + if err != nil { + t.Fatal(err) + } + metrics, found := nested(t, apply, "status", "metrics") + if !found { + t.Fatal("an evaluated candidate without metrics must still carry an empty metrics object: its presence marks it populated") + } + if object, ok := metrics.(map[string]any); !ok || len(object) != 0 { + t.Fatalf("metrics = %#v, want an empty object", metrics) } } func TestRunStatusApplyPayload(t *testing.T) { when := time.Date(2026, 10, 5, 21, 35, 12, 0, time.UTC) - status := RunStatus{Message: "m", HasProgress: true, Round: 2, Evaluated: 5, LastProgress: when, CandidateNames: []string{"x", "y"}} + status := RunStatus{Published: 2, LastProgress: when, CandidateNames: []string{"x", "y"}} apply := buildRunStatusApply(testNamespace, testRunName, "42", status) if apply.GetKind() != "DynamoGraphDeploymentRun" || apply.GetName() != testRunName { t.Fatalf("identity = %s %s", apply.GetKind(), apply.GetName()) @@ -807,8 +815,13 @@ func TestRunStatusApplyPayload(t *testing.T) { if got, _ := nested(t, apply, "status", "lastProgressTime"); got != "2026-10-05T21:35:12Z" { t.Fatalf("lastProgressTime = %v", got) } - if got, _ := nested(t, apply, "status", "progress", "round"); got != int64(2) { - t.Fatalf("round = %#v", got) + if got, _ := nested(t, apply, "status", "progress", "candidates", "published"); got != int64(2) { + t.Fatalf("published = %#v", got) + } + for _, field := range []string{"message", "provenance"} { + if _, found := nested(t, apply, "status", field); found { + t.Fatalf("status.%s does not exist in the API and must not be written", field) + } } refs, _ := nested(t, apply, "status", "candidateRefs") if list, ok := refs.([]any); !ok || len(list) != 2 { diff --git a/deploy/operator/internal/dgdrrunpublisher/writer.go b/deploy/operator/internal/dgdrrunpublisher/writer.go index f1b376c13235..8a02f1c61934 100644 --- a/deploy/operator/internal/dgdrrunpublisher/writer.go +++ b/deploy/operator/internal/dgdrrunpublisher/writer.go @@ -22,9 +22,8 @@ import ( "encoding/json" "time" + v1beta2 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta2" "github.com/ai-dynamo/dynamo/deploy/operator/internal/dgdcreconcile" - v1beta2 "github.com/ai-dynamo/dynamo/deploy/operator/internal/dgdrrunpublisher/placeholderapi" - "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" @@ -37,8 +36,9 @@ import ( type StoredCandidate struct { // ResourceVersion is the version a status write for this candidate is constrained to. ResourceVersion string - // Populated reports that the candidate already has its evaluation status, for example - // when a stale cache led the reconciler to apply a candidate that exists. + // Populated reports that the candidate already has its evaluation status (a metrics + // object, possibly empty), for example when a stale cache led the reconciler to apply a + // candidate that exists. Populated bool } @@ -49,7 +49,7 @@ type writer interface { // read after the apply. It must be idempotent: applying a candidate that already // exists changes nothing. CreateCandidate(ctx context.Context, run *v1beta2.DynamoGraphDeploymentRun, name string, candidate dgdcreconcile.DesiredCandidate) (StoredCandidate, error) - // PopulateStatus records the candidate's evaluation facts and marks it Evaluated. The + // PopulateStatus records the candidate's evaluation metrics. The // write is constrained to resourceVersion, so a stale view of the candidate fails // with a conflict instead of rewriting status that was already populated. PopulateStatus(ctx context.Context, name, resourceVersion string, candidate dgdcreconcile.DesiredCandidate) error @@ -69,8 +69,6 @@ type clientWriter struct { Namespace string RunName string RunUID string - // Now stamps the Evaluated condition; tests replace it. - Now func() time.Time } var _ writer = (*clientWriter)(nil) @@ -102,16 +100,12 @@ func (w *clientWriter) CreateCandidate(ctx context.Context, run *v1beta2.DynamoG } return StoredCandidate{ ResourceVersion: stored.ResourceVersion, - Populated: meta.IsStatusConditionTrue(stored.Status.Conditions, evaluatedCondition), + Populated: stored.Status.Metrics != nil, }, nil } func (w *clientWriter) PopulateStatus(ctx context.Context, name, resourceVersion string, candidate dgdcreconcile.DesiredCandidate) error { - now := time.Now - if w.Now != nil { - now = w.Now - } - object, err := buildCandidateStatusApply(w.Namespace, name, resourceVersion, candidate, now()) + object, err := buildCandidateStatusApply(w.Namespace, name, resourceVersion, candidate) if err != nil { return err } @@ -167,7 +161,7 @@ func buildCandidateApply(scheme *runtime.Scheme, namespace, runUID string, run * return nil, err } apply := &unstructured.Unstructured{Object: content} - apply.SetGroupVersionKind(v1beta2.GroupVersion.WithKind("DynamoGraphDeploymentCandidate")) + apply.SetGroupVersionKind(v1beta2.DynamoGraphDeploymentCandidateGVK) return apply, nil } @@ -180,32 +174,25 @@ func withResourceVersion(metadata map[string]any, resourceVersion string) map[st return metadata } -// buildCandidateStatusApply is the status payload of a candidate: the metrics and the -// Evaluated condition. Candidate status is written once and then stays immutable, so the -// payload carries the resourceVersion the "not populated yet" decision was based on. -func buildCandidateStatusApply(namespace, name, resourceVersion string, candidate dgdcreconcile.DesiredCandidate, now time.Time) (*unstructured.Unstructured, error) { +// buildCandidateStatusApply is the status payload of a candidate: the metrics. Metrics are +// always written, even when empty, because their presence is what marks the candidate as +// populated. Candidate status is written once and then stays immutable, so the payload +// carries the resourceVersion the "not populated yet" decision was based on. +func buildCandidateStatusApply(namespace, name, resourceVersion string, candidate dgdcreconcile.DesiredCandidate) (*unstructured.Unstructured, error) { metrics, err := normalizeJSON(candidate.Metrics) if err != nil { return nil, err } apply := &unstructured.Unstructured{Object: map[string]any{ "metadata": withResourceVersion(map[string]any{"namespace": namespace, "name": name}, resourceVersion), - "status": map[string]any{ - "metrics": metrics, - "conditions": []any{map[string]any{ - "type": evaluatedCondition, - "status": string(metav1.ConditionTrue), - "reason": evaluatedCondition, - "message": "evaluated by the Sweeper", - "lastTransitionTime": now.UTC().Format(time.RFC3339), - }}, - }, + "status": map[string]any{"metrics": metrics}, }} - apply.SetGroupVersionKind(v1beta2.GroupVersion.WithKind("DynamoGraphDeploymentCandidate")) + apply.SetGroupVersionKind(v1beta2.DynamoGraphDeploymentCandidateGVK) return apply, nil } -// buildRunStatusApply is the status payload of the run. It lists every publisher-owned +// buildRunStatusApply is the status payload of the run: the candidate references, the +// published-candidate count and the progress time. It lists every publisher-owned // field it wants to keep: under server-side apply a field this manager owns and omits // from the payload is removed. It carries the resourceVersion of the run the decision to // write was based on, so a decision made from a stale cached run conflicts instead of @@ -216,11 +203,8 @@ func buildRunStatusApply(namespace, runName, resourceVersion string, status RunS refs = append(refs, map[string]any{"name": name}) } fields := map[string]any{ - "message": status.Message, "candidateRefs": refs, - } - if status.HasProgress { - fields["progress"] = map[string]any{"round": int64(status.Round), "evaluated": int64(status.Evaluated)} + "progress": map[string]any{"candidates": map[string]any{"published": int64(status.Published)}}, } if !status.LastProgress.IsZero() { fields["lastProgressTime"] = status.LastProgress.UTC().Format(time.RFC3339) @@ -229,7 +213,7 @@ func buildRunStatusApply(namespace, runName, resourceVersion string, status RunS "metadata": withResourceVersion(map[string]any{"namespace": namespace, "name": runName}, resourceVersion), "status": fields, }} - apply.SetGroupVersionKind(v1beta2.GroupVersion.WithKind("DynamoGraphDeploymentRun")) + apply.SetGroupVersionKind(v1beta2.DynamoGraphDeploymentRunGVK) return apply } From 9b3d1d70b7c247cdaa2f35ef43fcc6c03b6581e4 Mon Sep 17 00:00:00 2001 From: devivasudevan <49675305+devivasudevan@users.noreply.github.com> Date: Wed, 21 Oct 2026 21:39:03 +0200 Subject: [PATCH 18/40] feat(operator): admission for v1beta2 DGDR requests Defaulting and validation for native v1beta2 DynamoGraphDeploymentRequests, on top of the conversion API (#13744). Admission runs through the shared v1beta1 webhooks (matchPolicy: Equivalent) and is routed by the original request's API version: a native v1beta2 request is decoded from spec.v1beta2, defaulted, validated and re-encoded; a foreign-version carrier is never mutated, direct carrier creates are rejected, and an unchanged foreign view can still be updated. Defaults: backends, workload (static ISL 1024, OSL 128, concurrency 10), objective (Optimize, Throughput), recommendation (maxCandidates 1), hardware discovery on CREATE only, and the operator's AISimulate image. The structural defaults are applied in admission, not the CRD schema, because a schema default would also be applied to an object that carries spec.v1beta1 and conversion would then reject it; only nested defaults stay in the schema. Explicit empty objects (workload.static: {}, objective.sla: {}) are kept. The AISimulate input is derived from the typed spec with derived defaults (request_count, sweep.max_trials, strict_sla) computed first and spec.aisimulate.parameters merged last, once. The base's early returns for requests carrying spec.v1beta2 are removed so those requests are validated. internal/runtimeversion is renamed internal/versionoverride. Tests: defaulting and routing unit tests, a CRD drift test (no top-level defaults; nested defaults equal the webhook's), admission envtest rows, an API-server test that carrier objects stay writable through every served version, and a Helm unit test for matchPolicy. Signed-off-by: devivasudevan <49675305+devivasudevan@users.noreply.github.com> --- .../operator/templates/deployment.yaml | 3 + .../templates/webhook-configuration.yaml | 2 + .../platform/components/operator/values.yaml | 3 + .../namespace_restriction_webhooks_test.yaml | 20 + .../dynamographdeploymentrequest_types.go | 29 +- deploy/operator/cmd/main.go | 30 +- ...dia.com_dynamographdeploymentrequests.yaml | 41 +- .../nvidia.com_dynamographdeploymentruns.yaml | 41 +- .../controller/aisimulate/defaults.go | 66 +++ .../internal/controller/aisimulate/doc.go | 36 ++ .../internal/controller/aisimulate/image.go | 40 ++ .../controller/aisimulate/image_test.go | 45 ++ .../internal/controller/aisimulate/input.go | 250 +++++++++ .../controller/aisimulate/input_test.go | 433 ++++++++++++++++ .../internal/controller/aisimulate/paths.go | 63 +++ ...dynamographdeploymentrequest_controller.go | 90 +--- .../internal/controller/suite_envtest_test.go | 16 + .../internal/dynamo/component_common.go | 4 +- .../dynamo/component_frontend_test.go | 10 +- .../dynamo/component_namespace_test.go | 12 +- .../internal/dynamo/component_worker_test.go | 18 +- deploy/operator/internal/dynamo/graph.go | 10 +- deploy/operator/internal/dynamo/hash.go | 6 +- .../internal/features/runtime/gate.go | 6 +- .../internal/features/runtime/gate_test.go | 12 +- .../internal/features/runtime/gates.go | 14 +- .../internal/gpu/hardware_discoverer.go | 140 +++++ .../internal/gpu/hardware_discoverer_test.go | 90 ++++ .../versionoverride.go} | 2 +- .../versionoverride_test.go} | 2 +- .../dynamographdeploymentrequest_handler.go | 18 +- ...namographdeploymentrequest_routing_test.go | 129 +++++ ...ographdeploymentrequest_v1beta1_options.go | 13 + .../dynamographdeploymentrequest_v1beta2.go | 196 +++++++ ...graphdeploymentrequest_v1beta2_crd_test.go | 92 ++++ ...ographdeploymentrequest_v1beta2_helpers.go | 159 ++++++ ...namographdeploymentrequest_v1beta2_test.go | 488 ++++++++++++++++++ .../internal/webhook/request_version.go | 48 ++ .../internal/webhook/request_version_test.go | 78 +++ .../operator/internal/webhook/setup/setup.go | 12 +- .../dynamographdeploymentrequest.go | 43 +- ...hdeploymentrequest_carrier_envtest_test.go | 75 +++ .../dynamographdeploymentrequest_routing.go | 68 +++ .../dynamographdeploymentrequest_v1beta2.go | 125 +++++ ...namographdeploymentrequest_v1beta2_test.go | 181 +++++++ ...ploymentrequest_validation_envtest_test.go | 389 +++++++++++++- .../validation/planner_runtime_features.go | 4 +- .../webhook/validation/shared_helpers.go | 6 +- .../webhook/validation/suite_envtest_test.go | 37 ++ .../additional-resources/api-reference-k8s.md | 24 +- .../kubernetes-api/full-api-reference.mdx | 30 +- 51 files changed, 3517 insertions(+), 232 deletions(-) create mode 100644 deploy/operator/internal/controller/aisimulate/defaults.go create mode 100644 deploy/operator/internal/controller/aisimulate/doc.go create mode 100644 deploy/operator/internal/controller/aisimulate/image.go create mode 100644 deploy/operator/internal/controller/aisimulate/image_test.go create mode 100644 deploy/operator/internal/controller/aisimulate/input.go create mode 100644 deploy/operator/internal/controller/aisimulate/input_test.go create mode 100644 deploy/operator/internal/controller/aisimulate/paths.go create mode 100644 deploy/operator/internal/gpu/hardware_discoverer.go create mode 100644 deploy/operator/internal/gpu/hardware_discoverer_test.go rename deploy/operator/internal/{runtimeversion/runtimeversion.go => versionoverride/versionoverride.go} (99%) rename deploy/operator/internal/{runtimeversion/runtimeversion_test.go => versionoverride/versionoverride_test.go} (99%) create mode 100644 deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_routing_test.go create mode 100644 deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta1_options.go create mode 100644 deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2.go create mode 100644 deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_crd_test.go create mode 100644 deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_helpers.go create mode 100644 deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_test.go create mode 100644 deploy/operator/internal/webhook/request_version.go create mode 100644 deploy/operator/internal/webhook/request_version_test.go create mode 100644 deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_carrier_envtest_test.go create mode 100644 deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_routing.go create mode 100644 deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_v1beta2.go create mode 100644 deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_v1beta2_test.go diff --git a/deploy/helm/charts/platform/components/operator/templates/deployment.yaml b/deploy/helm/charts/platform/components/operator/templates/deployment.yaml index 8a4ea7dd9080..adb853253277 100644 --- a/deploy/helm/charts/platform/components/operator/templates/deployment.yaml +++ b/deploy/helm/charts/platform/components/operator/templates/deployment.yaml @@ -92,6 +92,9 @@ spec: {{- with .Values.controllerManager.manager.dgdrDefaultImage }} - {{ printf "--dgdr-default-image=%s" . | quote }} {{- end }} + {{- with .Values.controllerManager.manager.dgdrAISimulateDefaultImage }} + - {{ printf "--dgdr-aisimulate-default-image=%s" . | quote }} + {{- end }} command: - /manager env: diff --git a/deploy/helm/charts/platform/components/operator/templates/webhook-configuration.yaml b/deploy/helm/charts/platform/components/operator/templates/webhook-configuration.yaml index 5f6880fe46e6..770627f942b3 100644 --- a/deploy/helm/charts/platform/components/operator/templates/webhook-configuration.yaml +++ b/deploy/helm/charts/platform/components/operator/templates/webhook-configuration.yaml @@ -150,6 +150,7 @@ webhooks: namespace: {{ .Release.Namespace }} path: /validate-nvidia-com-v1beta1-dynamographdeploymentrequest failurePolicy: {{ .Values.webhook.failurePolicy }} + matchPolicy: Equivalent name: vdynamographdeploymentrequest.kb.io {{- if .Values.webhook.namespaceSelector }} namespaceSelector: @@ -274,6 +275,7 @@ webhooks: namespace: {{ .Release.Namespace }} path: /mutate-nvidia-com-v1beta1-dynamographdeploymentrequest failurePolicy: {{ .Values.webhook.failurePolicy }} + matchPolicy: Equivalent name: mdynamographdeploymentrequestv1beta1.kb.io {{- if .Values.webhook.namespaceSelector }} namespaceSelector: diff --git a/deploy/helm/charts/platform/components/operator/values.yaml b/deploy/helm/charts/platform/components/operator/values.yaml index c526a93bebde..bbc3df59e7c7 100644 --- a/deploy/helm/charts/platform/components/operator/values.yaml +++ b/deploy/helm/charts/platform/components/operator/values.yaml @@ -115,6 +115,9 @@ controllerManager: # derives dynamo-planner:, which only exists for GA # releases. dgdrDefaultImage: "" + # AISimulate runner image put into v1beta2 DGDR spec.aisimulate.image when unset. + # Empty leaves it unset, and v1beta2 Requests without an image are rejected. + dgdrAISimulateDefaultImage: "" resources: limits: cpu: 1024m diff --git a/deploy/helm/charts/platform/tests/namespace_restriction_webhooks_test.yaml b/deploy/helm/charts/platform/tests/namespace_restriction_webhooks_test.yaml index af335066ea7a..42edcb584537 100644 --- a/deploy/helm/charts/platform/tests/namespace_restriction_webhooks_test.yaml +++ b/deploy/helm/charts/platform/tests/namespace_restriction_webhooks_test.yaml @@ -97,3 +97,23 @@ tests: asserts: - hasDocuments: count: 4 + + - it: matches DGDR requests through every served API version + template: charts/dynamo-operator/templates/webhook-configuration.yaml + asserts: + - equal: + path: webhooks[3].name + value: vdynamographdeploymentrequest.kb.io + documentIndex: 0 + - equal: + path: webhooks[3].matchPolicy + value: Equivalent + documentIndex: 0 + - equal: + path: webhooks[2].name + value: mdynamographdeploymentrequestv1beta1.kb.io + documentIndex: 1 + - equal: + path: webhooks[2].matchPolicy + value: Equivalent + documentIndex: 1 diff --git a/deploy/operator/api/v1beta2/dynamographdeploymentrequest_types.go b/deploy/operator/api/v1beta2/dynamographdeploymentrequest_types.go index 39887c2d6d17..2a6e007ac3d4 100644 --- a/deploy/operator/api/v1beta2/dynamographdeploymentrequest_types.go +++ b/deploy/operator/api/v1beta2/dynamographdeploymentrequest_types.go @@ -110,11 +110,11 @@ type StaticWorkloadSpec struct { // WorkloadSpec defines the synthetic traffic model used for every candidate evaluation. type WorkloadSpec struct { - // Static defines one of AISimulate's synthetic workload shapes. When omitted, - // it searches the full KV-load ratio range at ISL 4000 and OSL 1000. - // An explicit empty object opts out of this underlay. + // Static defines one of AISimulate's synthetic workload shapes. Its fields are not + // defaulted individually: the default belongs to spec.workload as a whole, so an + // explicit workload object keeps exactly the fields it names. // +optional - // +kubebuilder:default={isl: 4000, osl: 1000, kvLoadRatio: {0.0, 1.0}} + // +kubebuilder:default={isl: 1024, osl: 128, concurrency: 10} Static *StaticWorkloadSpec `json:"static,omitempty"` } @@ -176,24 +176,26 @@ type ObjectiveSpec struct { // +listType=set Metrics []ObjectiveMetric `json:"metrics,omitempty"` - // SLA defines per-request latency bounds used to calculate goodput. + // SLA defines per-request latency bounds used to calculate goodput. When omitted, no + // bounds apply. spec.aisimulate.parameters.goal.sla overrides the bounds it names. // +optional - // +kubebuilder:default={ttftMs: 2000, itlMs: 30} SLA *ObjectiveSLASpec `json:"sla,omitempty"` } // AISimulateSpec selects the AISimulate execution artifact and supplies native input. type AISimulateSpec struct { // Image selects a Dynamo-compatible AISimulate runner. When omitted, admission - // and execution defaults are resolved outside this API. + // persists the operator's configured image. // +optional // +kubebuilder:validation:MinLength=1 Image string `json:"image,omitempty"` - // VersionOverride declares the AISimulate compatibility version when it - // cannot be inferred from the effective image. + // VersionOverride declares the AISimulate compatibility version as three numeric parts + // (for example 0.13.0), not a semver version: no v prefix, pre-release or build suffix. It is required + // when the image tag is not a version and wins over a version in the tag. // +optional // +kubebuilder:validation:MinLength=1 + // +kubebuilder:validation:Pattern=`^(0|[1-9][0-9]{0,3})\.(0|[1-9][0-9]{0,3})\.(0|[1-9][0-9]{0,3})$` VersionOverride string `json:"versionOverride,omitempty"` // Parameters contains native AISimulate input. Field names and nesting use @@ -234,12 +236,17 @@ type NativeSearchInput struct { Hardware *HardwareSpec `json:"hardware,omitempty"` // Workload defines the synthetic traffic model used for every candidate evaluation. + // When omitted, admission supplies a static shape of ISL 1024, OSL 128 and concurrency 10; + // an explicit empty workload receives the same shape from the schema. An explicit + // workload.static: {} opts out of that shape, leaving the workload to + // spec.aisimulate.parameters.workload. spec.aisimulate.parameters.workload overrides the + // fields it names. // +optional Workload *WorkloadSpec `json:"workload,omitempty"` // Objective defines scalar optimization or a multi-objective Pareto search. - // When omitted, the schema supplies the default Optimize mode and SLA; - // admission supplies the conditional scalar metric. + // When omitted, admission supplies the object. Its mode defaults to Optimize, and admission + // supplies the conditional scalar metric. // +optional Objective *ObjectiveSpec `json:"objective,omitempty"` diff --git a/deploy/operator/cmd/main.go b/deploy/operator/cmd/main.go index 6081b6b5acea..8ec67cd11115 100644 --- a/deploy/operator/cmd/main.go +++ b/deploy/operator/cmd/main.go @@ -69,12 +69,14 @@ import ( "github.com/ai-dynamo/dynamo/deploy/operator/internal/crdmigrator" lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx/scheduler/v1alpha1" "github.com/ai-dynamo/dynamo/deploy/operator/internal/features" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/gpu" "github.com/ai-dynamo/dynamo/deploy/operator/internal/namespace_scope" "github.com/ai-dynamo/dynamo/deploy/operator/internal/observability" "github.com/ai-dynamo/dynamo/deploy/operator/internal/podcache" "github.com/ai-dynamo/dynamo/deploy/operator/internal/rbac" "github.com/ai-dynamo/dynamo/deploy/operator/internal/secret" "github.com/ai-dynamo/dynamo/deploy/operator/internal/secrets" + webhookdefaulting "github.com/ai-dynamo/dynamo/deploy/operator/internal/webhook/defaulting" webhooksetup "github.com/ai-dynamo/dynamo/deploy/operator/internal/webhook/setup" grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" snapshotv1alpha1 "github.com/ai-dynamo/snapshot/api/v1alpha1" @@ -117,6 +119,7 @@ func initCRDSchemes() { utilruntime.Must(nvidiacomv1alpha1.AddToScheme(crdScheme)) utilruntime.Must(nvidiacomv1beta1.AddToScheme(crdScheme)) + utilruntime.Must(nvidiacomv1beta2.AddToScheme(crdScheme)) utilruntime.Must(nvidiacomv1beta2.AddToScheme(crdScheme)) @@ -160,6 +163,7 @@ func main() { var operatorImage string var operatorImagePullPolicy string var dgdrDefaultImage string + var dgdrAISimulateDefaultImage string flag.StringVar(&configFile, "config", "", "Path to operator configuration file (required)") flag.StringVar(&operatorVersion, "operator-version", "unknown", "Version of the operator (used in lease holder identity)") @@ -173,6 +177,8 @@ func main() { "Image pull policy for operator helper init containers") flag.StringVar(&dgdrDefaultImage, "dgdr-default-image", "", "Default DGDR profiler image, put into DGDR spec.image when unset; empty derives dynamo-planner:") + flag.StringVar(&dgdrAISimulateDefaultImage, "dgdr-aisimulate-default-image", "", + "AISimulate runner image, put into v1beta2 DGDR spec.aisimulate.image when unset; empty leaves the image unset and the Request is rejected") opts := zap.Options{ Development: true, } @@ -505,7 +511,7 @@ func main() { } if err := registerWebhookHandlers( - mgr, operatorCfg, runtimeConfig, operatorVersion, dgdrDefaultImage, gates, + mgr, operatorCfg, runtimeConfig, operatorVersion, dgdrDefaultImage, dgdrAISimulateDefaultImage, gates, ); err != nil { setupLog.Error(err, "failed to register webhooks") os.Exit(1) @@ -645,6 +651,7 @@ func registerWebhookHandlers( runtimeConfig *commonController.RuntimeConfig, operatorVersion string, dgdrDefaultImage string, + dgdrAISimulateDefaultImage string, gate features.Gate, ) error { var operatorPrincipal string @@ -656,10 +663,23 @@ func registerWebhookHandlers( } if err := webhooksetup.Setup(mgr, webhooksetup.Options{ - Config: operatorCfg, - RuntimeConfig: runtimeConfig, - OperatorVersion: operatorVersion, - DGDRDefaultImage: dgdrDefaultImage, + Config: operatorCfg, + RuntimeConfig: runtimeConfig, + OperatorVersion: operatorVersion, + DGDRV1beta1: webhookdefaulting.DGDRV1beta1Options{ + DefaultImage: dgdrDefaultImage, + }, + DGDRV1beta2: webhookdefaulting.DGDRV1beta2Options{ + AISimulateImage: dgdrAISimulateDefaultImage, + // The same discovery order as the DGDR reconciler: DCGM first, then node labels when + // the GPUDiscovery gate is on. The webhook keeps its own short-lived cache. + Hardware: webhookdefaulting.NewGPUHardwareDiscoverer(&gpu.Discoverer{ + Reader: mgr.GetAPIReader(), + DCGM: gpu.NewGPUDiscovery(gpu.ScrapeMetricsEndpoint), + Cache: gpu.NewGPUDiscoveryCache(), + NodeLabels: func() bool { return gate.Enabled(features.GPUDiscovery) }, + }), + }, OperatorPrincipal: operatorPrincipal, Gate: gate, }); err != nil { diff --git a/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentrequests.yaml b/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentrequests.yaml index 404cc63ec777..0c4c79c78372 100644 --- a/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentrequests.yaml +++ b/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentrequests.yaml @@ -9594,7 +9594,7 @@ spec: image: description: |- Image selects a Dynamo-compatible AISimulate runner. When omitted, admission - and execution defaults are resolved outside this API. + persists the operator's configured image. minLength: 1 type: string parameters: @@ -9605,9 +9605,11 @@ spec: x-kubernetes-preserve-unknown-fields: true versionOverride: description: |- - VersionOverride declares the AISimulate compatibility version when it - cannot be inferred from the effective image. + VersionOverride declares the AISimulate compatibility version as three numeric parts + (for example 0.13.0), not a semver version: no v prefix, pre-release or build suffix. It is required + when the image tag is not a version and wins over a version in the tag. minLength: 1 + pattern: ^(0|[1-9][0-9]{0,3})\.(0|[1-9][0-9]{0,3})\.(0|[1-9][0-9]{0,3})$ type: string type: object backends: @@ -9659,8 +9661,8 @@ spec: objective: description: |- Objective defines scalar optimization or a multi-objective Pareto search. - When omitted, the schema supplies the default Optimize mode and SLA; - admission supplies the conditional scalar metric. + When omitted, admission supplies the object. Its mode defaults to Optimize, and admission + supplies the conditional scalar metric. properties: metric: description: |- @@ -9700,10 +9702,9 @@ spec: - Pareto type: string sla: - default: - itlMs: 30 - ttftMs: 2000 - description: SLA defines per-request latency bounds used to calculate goodput. + description: |- + SLA defines per-request latency bounds used to calculate goodput. When omitted, no + bounds apply. spec.aisimulate.parameters.goal.sla overrides the bounds it names. properties: e2eMs: description: E2EMs is an alternative end-to-end bound in milliseconds. @@ -9755,19 +9756,23 @@ spec: type: object x-kubernetes-preserve-unknown-fields: true workload: - description: Workload defines the synthetic traffic model used for every candidate evaluation. + description: |- + Workload defines the synthetic traffic model used for every candidate evaluation. + When omitted, admission supplies a static shape of ISL 1024, OSL 128 and concurrency 10; + an explicit empty workload receives the same shape from the schema. An explicit + workload.static: {} opts out of that shape, leaving the workload to + spec.aisimulate.parameters.workload. spec.aisimulate.parameters.workload overrides the + fields it names. properties: static: default: - isl: 4000 - kvLoadRatio: - - 0 - - 1 - osl: 1000 + concurrency: 10 + isl: 1024 + osl: 128 description: |- - Static defines one of AISimulate's synthetic workload shapes. When omitted, - it searches the full KV-load ratio range at ISL 4000 and OSL 1000. - An explicit empty object opts out of this underlay. + Static defines one of AISimulate's synthetic workload shapes. Its fields are not + defaulted individually: the default belongs to spec.workload as a whole, so an + explicit workload object keeps exactly the fields it names. properties: concurrency: description: Concurrency is a fixed closed-loop in-flight request cap. diff --git a/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentruns.yaml b/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentruns.yaml index 70a8e6f5e736..eb90a63fc2ac 100644 --- a/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentruns.yaml +++ b/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentruns.yaml @@ -66,7 +66,7 @@ spec: image: description: |- Image selects a Dynamo-compatible AISimulate runner. When omitted, admission - and execution defaults are resolved outside this API. + persists the operator's configured image. minLength: 1 type: string parameters: @@ -77,9 +77,11 @@ spec: x-kubernetes-preserve-unknown-fields: true versionOverride: description: |- - VersionOverride declares the AISimulate compatibility version when it - cannot be inferred from the effective image. + VersionOverride declares the AISimulate compatibility version as three numeric parts + (for example 0.13.0), not a semver version: no v prefix, pre-release or build suffix. It is required + when the image tag is not a version and wins over a version in the tag. minLength: 1 + pattern: ^(0|[1-9][0-9]{0,3})\.(0|[1-9][0-9]{0,3})\.(0|[1-9][0-9]{0,3})$ type: string type: object backends: @@ -131,8 +133,8 @@ spec: objective: description: |- Objective defines scalar optimization or a multi-objective Pareto search. - When omitted, the schema supplies the default Optimize mode and SLA; - admission supplies the conditional scalar metric. + When omitted, admission supplies the object. Its mode defaults to Optimize, and admission + supplies the conditional scalar metric. properties: metric: description: |- @@ -172,10 +174,9 @@ spec: - Pareto type: string sla: - default: - itlMs: 30 - ttftMs: 2000 - description: SLA defines per-request latency bounds used to calculate goodput. + description: |- + SLA defines per-request latency bounds used to calculate goodput. When omitted, no + bounds apply. spec.aisimulate.parameters.goal.sla overrides the bounds it names. properties: e2eMs: description: E2EMs is an alternative end-to-end bound in milliseconds. @@ -219,19 +220,23 @@ spec: type: integer type: object workload: - description: Workload defines the synthetic traffic model used for every candidate evaluation. + description: |- + Workload defines the synthetic traffic model used for every candidate evaluation. + When omitted, admission supplies a static shape of ISL 1024, OSL 128 and concurrency 10; + an explicit empty workload receives the same shape from the schema. An explicit + workload.static: {} opts out of that shape, leaving the workload to + spec.aisimulate.parameters.workload. spec.aisimulate.parameters.workload overrides the + fields it names. properties: static: default: - isl: 4000 - kvLoadRatio: - - 0 - - 1 - osl: 1000 + concurrency: 10 + isl: 1024 + osl: 128 description: |- - Static defines one of AISimulate's synthetic workload shapes. When omitted, - it searches the full KV-load ratio range at ISL 4000 and OSL 1000. - An explicit empty object opts out of this underlay. + Static defines one of AISimulate's synthetic workload shapes. Its fields are not + defaulted individually: the default belongs to spec.workload as a whole, so an + explicit workload object keeps exactly the fields it names. properties: concurrency: description: Concurrency is a fixed closed-loop in-flight request cap. diff --git a/deploy/operator/internal/controller/aisimulate/defaults.go b/deploy/operator/internal/controller/aisimulate/defaults.go new file mode 100644 index 000000000000..5631a0eded41 --- /dev/null +++ b/deploy/operator/internal/controller/aisimulate/defaults.go @@ -0,0 +1,66 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package aisimulate + +import "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + +// The derived defaults. No API field carries them: each depends on the typed underlay, so +// no single typed field can supply it. +const ( + // DefaultRequestCount is workload.request_count when the typed underlay does not + // choose num_request_ratio. + DefaultRequestCount int64 = 1000 + + // DefaultMaxTrials bounds a KV-load range search that has no explicit sweep.max_trials. + // It matches the trial budget of AISimulate's unified CLI (optimizer.max_trials). + DefaultMaxTrials int64 = 320 +) + +// applyDefaults adds the derived defaults to the typed underlay, before the native +// parameters are applied. spec.aisimulate.parameters is the final overlay: it overrides or +// deletes any derived value, and nothing is derived in response to what it introduces. +// A native change that needs a matching change elsewhere (a request_count next to +// num_request_ratio, a strict_sla next to a new SLA bound) must make it itself. +func applyDefaults(underlay map[string]any) error { + b := builder{root: underlay} + + if has(underlay, Workload...) { + if !has(underlay, WorkloadRequestCount...) && !has(underlay, WorkloadNumRequestRatio...) { + b.set(float64(DefaultRequestCount), WorkloadRequestCount) + } + kvLoadRatio, _, _ := unstructured.NestedFieldNoCopy(underlay, WorkloadKVLoadRatio...) + if _, isRange := kvLoadRatio.([]any); isRange && !has(underlay, SweepMaxTrials...) { + b.set(float64(DefaultMaxTrials), SweepMaxTrials) + } + } + + if hasSLABound(underlay) && !has(underlay, GoalStrictSLA...) { + b.set(true, GoalStrictSLA) + } + return b.err +} + +// hasSLABound reports whether the input bounds at least one latency. +func hasSLABound(input map[string]any) bool { + for _, path := range [][]string{GoalSLATTFTMs, GoalSLAITLMs, GoalSLAE2EMs} { + if bound, _, _ := unstructured.NestedFieldNoCopy(input, path...); bound != nil { + return true + } + } + return false +} diff --git a/deploy/operator/internal/controller/aisimulate/doc.go b/deploy/operator/internal/controller/aisimulate/doc.go new file mode 100644 index 000000000000..07d398d60e78 --- /dev/null +++ b/deploy/operator/internal/controller/aisimulate/doc.go @@ -0,0 +1,36 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package aisimulate lowers a v1beta2 DynamoGraphDeploymentRequest spec into the native +// AISimulate input that a Run executes. +// +// Lowering sits below the Request reconciler. Admission calls it as a second validation +// stage, after the typed spec has been defaulted and validated on its own, and the +// reconciler freezes its output into the Run, so both see the same bytes. +// +// The typed spec (model, backends, hardware, workload, objective) is first translated +// into the native input, the "underlay". spec.aisimulate.parameters is then applied on +// top as a JSON Merge Patch (RFC 7386): native objects merge recursively, arrays and +// scalars replace the underlay value, and null deletes it. The native input therefore +// wins on every path it names. Finally the derived defaults that no API field carries are +// added: the request count, a bounded trial budget for a KV-load range search, and +// goal.strict_sla. +// +// The document stays unstructured (map[string]any) because its schema belongs to the +// AISimulate release the Request selects, not to this package. Reads and writes go +// through the helpers of k8s.io/apimachinery/pkg/apis/meta/v1/unstructured. +package aisimulate diff --git a/deploy/operator/internal/controller/aisimulate/image.go b/deploy/operator/internal/controller/aisimulate/image.go new file mode 100644 index 000000000000..91afdcb4496f --- /dev/null +++ b/deploy/operator/internal/controller/aisimulate/image.go @@ -0,0 +1,40 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package aisimulate + +import ( + "errors" + "fmt" + + "github.com/ai-dynamo/dynamo/deploy/operator/internal/versionoverride" +) + +// ResolveVersion returns the AISimulate compatibility version of a Request: the explicit +// override when set, otherwise the version in the image tag. An image whose tag is not a +// version, or a missing image, is an error; the caller reports it on the field that fixes it. +// The parsing is shared with the runtime version of the other Request kinds. +func ResolveVersion(image, versionOverride string) (string, error) { + if image == "" && versionOverride == "" { + return "", errors.New("no AISimulate image") + } + version, err := versionoverride.Resolve(image, versionOverride) + if err != nil { + return "", fmt.Errorf("resolve AISimulate version: %w", err) + } + return version.String(), nil +} diff --git a/deploy/operator/internal/controller/aisimulate/image_test.go b/deploy/operator/internal/controller/aisimulate/image_test.go new file mode 100644 index 000000000000..064c3b80b2a8 --- /dev/null +++ b/deploy/operator/internal/controller/aisimulate/image_test.go @@ -0,0 +1,45 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package aisimulate + +import "testing" + +func TestResolveVersion(t *testing.T) { + cases := []struct { + name, image, override, want string + wantErr bool + }{ + {name: "version from the tag", image: "registry.example.com/aisimulate:1.2.3", want: "1.2.3"}, + {name: "v-prefixed tag", image: "registry.example.com/aisimulate:v1.2.3", want: "1.2.3"}, + {name: "override wins over the tag", image: "registry.example.com/aisimulate:1.2.3", override: "2.0.0", want: "2.0.0"}, + {name: "override rescues a custom tag", image: "registry.example.com/aisimulate:custom", override: "1.4.0", want: "1.4.0"}, + {name: "custom tag needs an override", image: "registry.example.com/aisimulate:custom", wantErr: true}, + {name: "untagged image needs an override", image: "registry.example.com/aisimulate", wantErr: true}, + {name: "registry port is not a tag", image: "registry.example.com:5000/aisimulate", wantErr: true}, + {name: "no image", wantErr: true}, + {name: "override without an image", override: "1.0.0", want: "1.0.0"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got, err := ResolveVersion(tc.image, tc.override) + if (err != nil) != tc.wantErr || got != tc.want { + t.Fatalf("ResolveVersion(%q, %q) = %q, %v; want %q (error %v)", tc.image, tc.override, got, err, tc.want, tc.wantErr) + } + }) + } +} diff --git a/deploy/operator/internal/controller/aisimulate/input.go b/deploy/operator/internal/controller/aisimulate/input.go new file mode 100644 index 000000000000..0b8bfc179bb6 --- /dev/null +++ b/deploy/operator/internal/controller/aisimulate/input.go @@ -0,0 +1,250 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package aisimulate + +import ( + "encoding/json" + "fmt" + "strings" + "unicode" + + v1beta2 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta2" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" +) + +// Lower returns the effective native input: the typed underlay, merged with +// spec.aisimulate.parameters, plus the derived defaults. The spec must already carry its +// structural defaults; lowering repeats none of them. The only error is malformed +// parameters (not a JSON object); semantic problems are the caller's to validate. +func Lower(spec *v1beta2.DynamoGraphDeploymentRequestSpec) (map[string]any, error) { + parameters, err := Parameters(spec) + if err != nil { + return nil, err + } + underlay, err := Underlay(spec) + if err != nil { + return nil, err + } + if underlay, err = normalize(underlay); err != nil { + return nil, err + } + if err := applyDefaults(underlay); err != nil { + return nil, err + } + input, _ := mergePatch(underlay, parameters).(map[string]any) + if input == nil { + input = map[string]any{} + } + return input, nil +} + +// Parameters returns spec.aisimulate.parameters as a JSON object, or nil when absent. +func Parameters(spec *v1beta2.DynamoGraphDeploymentRequestSpec) (map[string]any, error) { + if spec.AISimulate == nil || spec.AISimulate.Parameters == nil { + return nil, nil + } + raw := spec.AISimulate.Parameters.Raw + if len(raw) == 0 && spec.AISimulate.Parameters.Object != nil { + var err error + if raw, err = json.Marshal(spec.AISimulate.Parameters.Object); err != nil { + return nil, fmt.Errorf("spec.aisimulate.parameters: %w", err) + } + } + if len(raw) == 0 { + return nil, nil + } + var value any + if err := json.Unmarshal(raw, &value); err != nil { + return nil, fmt.Errorf("spec.aisimulate.parameters: %w", err) + } + switch typed := value.(type) { + case nil: + return nil, nil + case map[string]any: + return typed, nil + default: + return nil, fmt.Errorf("spec.aisimulate.parameters: must be a JSON object, got %T", value) + } +} + +// Underlay translates the typed spec into native input. Typed fields that are unset +// produce no key, so the native defaults and the parameters decide them. +func Underlay(spec *v1beta2.DynamoGraphDeploymentRequestSpec) (map[string]any, error) { + b := builder{root: map[string]any{}} + + if spec.Model != nil { + b.set(spec.Model.Name, SearchSpaceModelName) + } + if len(spec.Backends) > 0 { + backends := make([]string, len(spec.Backends)) + for i, backend := range spec.Backends { + backends[i] = string(backend) + } + b.setStrings(backends, SearchSpaceBackend) + } + if spec.Hardware != nil && spec.Hardware.GPU != nil { + if sku := spec.Hardware.GPU.SKU; sku != nil && *sku != "" { + b.set(*sku, SearchSpaceHardwareSKU) + } + if budget := spec.Hardware.GPU.Budget; budget != nil { + b.set(int64(*budget), SearchSpaceGPUBudget) + } + } + + if spec.Workload != nil && spec.Workload.Static != nil { + static := spec.Workload.Static + if static.ISL != nil { + b.set(int64(*static.ISL), WorkloadISL) + } + if static.OSL != nil { + b.set(int64(*static.OSL), WorkloadOSL) + } + if static.NumRequestRatio != nil { + b.set(*static.NumRequestRatio, WorkloadNumRequestRatio) + } + if static.RequestRate != nil { + b.set(*static.RequestRate, WorkloadRequestRate) + } + if static.Concurrency != nil { + b.set(int64(*static.Concurrency), WorkloadConcurrency) + } + switch len(static.KVLoadRatio) { + case 0: + case 1: + b.set(static.KVLoadRatio[0], WorkloadKVLoadRatio) + default: + ratios := make([]any, len(static.KVLoadRatio)) + for i, ratio := range static.KVLoadRatio { + ratios[i] = ratio + } + b.set(ratios, WorkloadKVLoadRatio) + } + } + + if objective := spec.Objective; objective != nil { + if objective.Mode == v1beta2.ObjectiveModePareto { + b.set(TargetPareto, GoalTarget) + if len(objective.Metrics) > 0 { + b.setStrings(nativeMetrics(objective.Metrics), GoalParetoObjectives) + } + } else if objective.Metric != nil { + b.set(nativeMetric(*objective.Metric), GoalTarget) + } + if sla := objective.SLA; sla != nil { + if sla.TTFTMs != nil { + b.set(*sla.TTFTMs, GoalSLATTFTMs) + } + if sla.ITLMs != nil { + b.set(*sla.ITLMs, GoalSLAITLMs) + } + if sla.E2EMs != nil { + b.set(*sla.E2EMs, GoalSLAE2EMs) + } + } + } + return b.root, b.err +} + +// builder writes native input values, remembering the first failure so the translation reads +// as a straight list of assignments. Every path is written under a map the caller created +// and every value is a JSON scalar or a list of them, so a failure means a bad path. +type builder struct { + root map[string]any + err error +} + +func (b *builder) set(value any, path []string) { + if b.err != nil { + return + } + if err := unstructured.SetNestedField(b.root, value, path...); err != nil { + b.err = fmt.Errorf("set %s: %w", strings.Join(path, "."), err) + } +} + +func (b *builder) setStrings(values []string, path []string) { + if b.err != nil { + return + } + if err := unstructured.SetNestedStringSlice(b.root, values, path...); err != nil { + b.err = fmt.Errorf("set %s: %w", strings.Join(path, "."), err) + } +} + +// nativeMetric maps an API metric to AISimulate's goal.target by converting its CamelCase +// name to snake_case: ThroughputPerGPU becomes throughput_per_gpu and E2ELatency becomes +// e2e_latency. A run of capitals stays together, a digit does not start a word, and the last +// capital of a run starts the next word when a lowercase letter follows it. +func nativeMetric(metric v1beta2.ObjectiveMetric) string { + name := []rune(string(metric)) + var out []rune + for i, r := range name { + if i > 0 && unicode.IsUpper(r) { + prev := name[i-1] + nextLower := i+1 < len(name) && unicode.IsLower(name[i+1]) + if unicode.IsLower(prev) || (unicode.IsUpper(prev) && nextLower) { + out = append(out, '_') + } + } + out = append(out, unicode.ToLower(r)) + } + return string(out) +} + +func nativeMetrics(metrics []v1beta2.ObjectiveMetric) []string { + out := make([]string, len(metrics)) + for i, metric := range metrics { + out[i] = nativeMetric(metric) + } + return out +} + +// normalize round-trips through JSON so every number is a float64, the type the +// parameters decode to. Merge and validation then see one representation. +func normalize(value map[string]any) (map[string]any, error) { + raw, err := json.Marshal(value) + if err != nil { + return nil, err + } + var out map[string]any + if err := json.Unmarshal(raw, &out); err != nil { + return nil, err + } + return out, nil +} + +// mergePatch applies patch to target with RFC 7386 semantics. It returns a new value for +// the merged parts and does not retain the patch's maps for null members. +func mergePatch(target, patch any) any { + patchMap, ok := patch.(map[string]any) + if !ok { + return patch + } + targetMap, ok := target.(map[string]any) + if !ok { + targetMap = map[string]any{} + } + for key, value := range patchMap { + if value == nil { + delete(targetMap, key) + continue + } + targetMap[key] = mergePatch(targetMap[key], value) + } + return targetMap +} diff --git a/deploy/operator/internal/controller/aisimulate/input_test.go b/deploy/operator/internal/controller/aisimulate/input_test.go new file mode 100644 index 000000000000..dfeff5a150a0 --- /dev/null +++ b/deploy/operator/internal/controller/aisimulate/input_test.go @@ -0,0 +1,433 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package aisimulate + +import ( + "encoding/json" + "strings" + "testing" + + v1beta2 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta2" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" +) + +func ptrTo[T any](v T) *T { return &v } + +// defaultedSpec is a model-only Request as it looks after the CRD's structural defaults and +// admission: the fixed-concurrency workload, the scalar metric and discovered hardware. It +// has no SLA bounds and no KV-load range; withSLA and withKVRange add them. +func defaultedSpec() *v1beta2.DynamoGraphDeploymentRequestSpec { + metric := v1beta2.ObjectiveMetric("Throughput") + return &v1beta2.DynamoGraphDeploymentRequestSpec{ + Model: &v1beta2.ModelSpec{Name: "meta-llama/Llama-3.1-8B-Instruct"}, + Backends: []v1beta2.Backend{v1beta2.BackendVLLM, v1beta2.BackendSGLang, v1beta2.BackendTRTLLM}, + Hardware: &v1beta2.HardwareSpec{GPU: &v1beta2.GPUHardwareSpec{SKU: ptrTo("h200_sxm"), Budget: ptrTo(int32(8))}}, + Workload: &v1beta2.WorkloadSpec{Static: &v1beta2.StaticWorkloadSpec{ + ISL: ptrTo(int32(1024)), OSL: ptrTo(int32(128)), Concurrency: ptrTo(int32(10)), + }}, + Objective: &v1beta2.ObjectiveSpec{Mode: v1beta2.ObjectiveModeOptimize, Metric: &metric}, + Recommendation: &v1beta2.RecommendationSpec{MaxCandidates: 1}, + } +} + +// withSLA adds the latency bounds the earlier defaults used to supply. +func withSLA(spec *v1beta2.DynamoGraphDeploymentRequestSpec) *v1beta2.DynamoGraphDeploymentRequestSpec { + spec.Objective.SLA = &v1beta2.ObjectiveSLASpec{TTFTMs: ptrTo(2000.0), ITLMs: ptrTo(30.0)} + return spec +} + +// withKVRange replaces the fixed concurrency with a KV-load range search. +func withKVRange(spec *v1beta2.DynamoGraphDeploymentRequestSpec) *v1beta2.DynamoGraphDeploymentRequestSpec { + spec.Workload.Static = &v1beta2.StaticWorkloadSpec{ + ISL: ptrTo(int32(4000)), OSL: ptrTo(int32(1000)), KVLoadRatio: v1beta2.KVLoadRatio{0, 1}, + } + return spec +} + +func withParameters(spec *v1beta2.DynamoGraphDeploymentRequestSpec, parameters string) *v1beta2.DynamoGraphDeploymentRequestSpec { + spec.AISimulate = &v1beta2.AISimulateSpec{Parameters: &runtime.RawExtension{Raw: []byte(parameters)}} + return spec +} + +func lower(t *testing.T, spec *v1beta2.DynamoGraphDeploymentRequestSpec) map[string]any { + t.Helper() + input, err := Lower(spec) + if err != nil { + t.Fatalf("Lower: %v", err) + } + return input +} + +func encode(t *testing.T, input map[string]any) string { + t.Helper() + raw, err := json.Marshal(input) + if err != nil { + t.Fatal(err) + } + return string(raw) +} + +func underlay(t *testing.T, spec *v1beta2.DynamoGraphDeploymentRequestSpec) map[string]any { + t.Helper() + input, err := Underlay(spec) + if err != nil { + t.Fatalf("Underlay: %v", err) + } + return input +} + +func nested(t *testing.T, input map[string]any, path ...string) any { + t.Helper() + value, _, err := unstructured.NestedFieldNoCopy(input, path...) + if err != nil { + t.Fatal(err) + } + return value +} + +func TestLowerModelOnlyRequest(t *testing.T) { + want := `{"goal":{"target":"throughput"},` + + `"search_space":{"backend":["vllm","sglang","trtllm"],"gpu_budget":8,"hardware_sku":"h200_sxm","model_name":"meta-llama/Llama-3.1-8B-Instruct"},` + + `"workload":{"concurrency":10,"isl":1024,"osl":128,"request_count":1000}}` + if got := encode(t, lower(t, defaultedSpec())); got != want { + t.Fatalf("effective input\n got: %s\nwant: %s", got, want) + } +} + +func TestLowerIsDeterministicAndDoesNotMutateTheSpec(t *testing.T) { + const parameters = `{"sweep":{"max_trials":100},"workload":{"request_count":5}}` + spec := withParameters(defaultedSpec(), parameters) + first, second := encode(t, lower(t, spec)), encode(t, lower(t, spec)) + if first != second { + t.Fatalf("not deterministic:\n%s\n%s", first, second) + } + if string(spec.AISimulate.Parameters.Raw) != parameters { + t.Fatalf("parameters mutated: %s", spec.AISimulate.Parameters.Raw) + } +} + +func TestLowerExplicitRequest(t *testing.T) { + metric := v1beta2.ObjectiveMetric("Throughput") + spec := withSLA(defaultedSpec()) + spec.Backends = []v1beta2.Backend{v1beta2.BackendVLLM, v1beta2.BackendSGLang} + spec.Workload.Static = &v1beta2.StaticWorkloadSpec{ISL: ptrTo(int32(4000)), OSL: ptrTo(int32(1000)), RequestRate: ptrTo(10.0)} + spec.Objective.Metric = &metric + spec = withParameters(spec, `{"workload":{"request_count":1000},"sweep":{"max_trials":100}}`) + + want := `{"goal":{"sla":{"itl_ms":30,"ttft_ms":2000},"strict_sla":true,"target":"throughput"},` + + `"search_space":{"backend":["vllm","sglang"],"gpu_budget":8,"hardware_sku":"h200_sxm","model_name":"meta-llama/Llama-3.1-8B-Instruct"},` + + `"sweep":{"max_trials":100},` + + `"workload":{"isl":4000,"osl":1000,"request_count":1000,"request_rate":10}}` + if got := encode(t, lower(t, spec)); got != want { + t.Fatalf("effective input\n got: %s\nwant: %s", got, want) + } +} + +func TestMergePatchSemantics(t *testing.T) { + cases := []struct{ name, target, patch, want string }{ + {"scalar replaces", `{"a":1}`, `{"a":2}`, `{"a":2}`}, + {"object merges recursively", `{"a":{"b":1,"c":2}}`, `{"a":{"b":3}}`, `{"a":{"b":3,"c":2}}`}, + {"null deletes", `{"a":1,"b":2}`, `{"a":null}`, `{"b":2}`}, + {"array replaces", `{"a":[1,2,3]}`, `{"a":[4]}`, `{"a":[4]}`}, + {"object replaces scalar and drops nulls", `{"a":1}`, `{"a":{"b":null,"c":1}}`, `{"a":{"c":1}}`}, + {"null on absent key is a no-op", `{"a":1}`, `{"z":null}`, `{"a":1}`}, + {"empty patch", `{"a":1}`, `{}`, `{"a":1}`}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + var target, patch any + if err := json.Unmarshal([]byte(tc.target), &target); err != nil { + t.Fatal(err) + } + if err := json.Unmarshal([]byte(tc.patch), &patch); err != nil { + t.Fatal(err) + } + got, _ := json.Marshal(mergePatch(target, patch)) + if string(got) != tc.want { + t.Fatalf("got %s want %s", got, tc.want) + } + }) + } + t.Run("nil patch map", func(t *testing.T) { + target := map[string]any{"a": 1.0} + if got := mergePatch(target, map[string]any(nil)); got == nil { + t.Fatal("a nil patch must keep the target") + } + }) +} + +func TestNativeValuesWinOverTheTypedUnderlay(t *testing.T) { + t.Run("a native null deletes a typed field", func(t *testing.T) { + input := lower(t, withParameters(defaultedSpec(), `{"workload":{"isl":null}}`)) + if nested(t, input, "workload", "isl") != nil { + t.Fatalf("isl survived a native null: %s", encode(t, input)) + } + }) + t.Run("a native value overrides a typed field", func(t *testing.T) { + input := lower(t, withParameters(defaultedSpec(), `{"workload":{"isl":8000},"search_space":{"gpu_budget":16}}`)) + if nested(t, input, "workload", "isl") != 8000.0 || nested(t, input, "search_space", "gpu_budget") != 16.0 { + t.Fatalf("native values did not win: %s", encode(t, input)) + } + }) + t.Run("a native target overrides the typed metric", func(t *testing.T) { + input := lower(t, withParameters(defaultedSpec(), `{"goal":{"target":"ttft"}}`)) + if nested(t, input, "goal", "target") != "ttft" { + t.Fatalf("typed metric beat the native target: %s", encode(t, input)) + } + }) + t.Run("a native SLA bound overrides only the bound it names", func(t *testing.T) { + input := lower(t, withParameters(withSLA(defaultedSpec()), `{"goal":{"sla":{"ttft_ms":500}}}`)) + if nested(t, input, "goal", "sla", "ttft_ms") != 500.0 || nested(t, input, "goal", "sla", "itl_ms") != 30.0 { + t.Fatalf("unexpected SLA: %s", encode(t, input)) + } + }) +} + +func TestRequestCountDefault(t *testing.T) { + t.Run("derived from the typed underlay", func(t *testing.T) { + if nested(t, lower(t, defaultedSpec()), "workload", "request_count") != 1000.0 { + t.Fatal("request_count not derived") + } + }) + t.Run("a native ratio deletes the derived count itself", func(t *testing.T) { + input := lower(t, withParameters(defaultedSpec(), `{"workload":{"num_request_ratio":10,"request_count":null}}`)) + if nested(t, input, "workload", "request_count") != nil || nested(t, input, "workload", "num_request_ratio") != 10.0 { + t.Fatalf("unexpected workload: %s", encode(t, input)) + } + }) + t.Run("skipped when typed numRequestRatio is set", func(t *testing.T) { + spec := defaultedSpec() + spec.Workload.Static.NumRequestRatio = ptrTo(5.0) + if nested(t, lower(t, spec), "workload", "request_count") != nil { + t.Fatal("request_count defaulted next to typed numRequestRatio") + } + }) + t.Run("native null deletes the default", func(t *testing.T) { + input := lower(t, withParameters(defaultedSpec(), `{"workload":{"request_count":null}}`)) + if nested(t, input, "workload", "request_count") != nil { + t.Fatal("request_count re-added after a native null") + } + }) + t.Run("native deleting the typed ratio does not derive a count", func(t *testing.T) { + spec := defaultedSpec() + spec.Workload.Static.NumRequestRatio = ptrTo(5.0) + input := lower(t, withParameters(spec, `{"workload":{"num_request_ratio":null}}`)) + if nested(t, input, "workload", "request_count") != nil { + t.Fatalf("a count was derived after the native overlay: %s", encode(t, input)) + } + }) + t.Run("not added without a workload", func(t *testing.T) { + spec := defaultedSpec() + spec.Workload = &v1beta2.WorkloadSpec{} + if nested(t, lower(t, spec), "workload") != nil { + t.Fatal("a workload appeared from nothing") + } + }) +} + +func TestMaxTrialsDefault(t *testing.T) { + t.Run("added for a KV-load range", func(t *testing.T) { + if nested(t, lower(t, withKVRange(defaultedSpec())), "sweep", "max_trials") != 320.0 { + t.Fatal("no default max_trials for a range search") + } + }) + t.Run("not derived for the model-only fixed concurrency", func(t *testing.T) { + if input := lower(t, defaultedSpec()); nested(t, input, "sweep") != nil { + t.Fatalf("unexpected sweep: %s", encode(t, input)) + } + }) + t.Run("not added for a pinned load", func(t *testing.T) { + spec := withKVRange(defaultedSpec()) + spec.Workload.Static.KVLoadRatio = v1beta2.KVLoadRatio{0.5} + input := lower(t, spec) + if nested(t, input, "sweep") != nil { + t.Fatalf("unexpected sweep: %s", encode(t, input)) + } + if nested(t, input, "workload", "kv_load_ratio") != 0.5 { + t.Fatal("a one-value ratio should be a scalar") + } + }) + t.Run("native value or null wins", func(t *testing.T) { + if nested(t, lower(t, withParameters(withKVRange(defaultedSpec()), `{"sweep":{"max_trials":7}}`)), "sweep", "max_trials") != 7.0 { + t.Fatal("native max_trials ignored") + } + if nested(t, lower(t, withParameters(withKVRange(defaultedSpec()), `{"sweep":{"max_trials":null}}`)), "sweep", "max_trials") != nil { + t.Fatal("max_trials re-added after a native null") + } + }) +} + +func TestStrictSLADerivation(t *testing.T) { + t.Run("enabled when the typed SLA has a bound", func(t *testing.T) { + if nested(t, lower(t, withSLA(defaultedSpec())), "goal", "strict_sla") != true { + t.Fatal("strict_sla not derived") + } + }) + t.Run("model-only derives nothing", func(t *testing.T) { + input := lower(t, defaultedSpec()) + if nested(t, input, "goal", "strict_sla") != nil || nested(t, input, "goal", "sla") != nil { + t.Fatalf("model-only input has SLA keys: %s", encode(t, input)) + } + }) + t.Run("an empty SLA derives nothing", func(t *testing.T) { + spec := defaultedSpec() + spec.Objective.SLA = &v1beta2.ObjectiveSLASpec{} + input := lower(t, spec) + if nested(t, input, "goal", "strict_sla") != nil || nested(t, input, "goal", "sla") != nil { + t.Fatalf("an empty SLA produced SLA keys: %s", encode(t, input)) + } + }) + t.Run("a native bound alone does not derive it", func(t *testing.T) { + input := lower(t, withParameters(defaultedSpec(), `{"goal":{"sla":{"ttft_ms":500}}}`)) + if nested(t, input, "goal", "sla", "ttft_ms") != 500.0 || nested(t, input, "goal", "strict_sla") != nil { + t.Fatalf("unexpected goal: %s", encode(t, input)) + } + }) + t.Run("native value is kept", func(t *testing.T) { + if nested(t, lower(t, withParameters(withSLA(defaultedSpec()), `{"goal":{"strict_sla":false}}`)), "goal", "strict_sla") != false { + t.Fatal("native strict_sla overridden") + } + }) + t.Run("is not revisited after the native overlay", func(t *testing.T) { + input := lower(t, withParameters(withSLA(defaultedSpec()), `{"goal":{"sla":{"ttft_ms":null,"itl_ms":null}}}`)) + if nested(t, input, "goal", "strict_sla") != true { + t.Fatalf("the derived strict_sla changed after native bounds were deleted: %s", encode(t, input)) + } + }) +} + +func TestParametersApplyOnceAfterDerivedDefaults(t *testing.T) { + t.Run("sweep null removes the complete derived sweep", func(t *testing.T) { + input := lower(t, withParameters(withKVRange(defaultedSpec()), `{"sweep":null}`)) + if nested(t, input, "sweep") != nil { + t.Fatalf("sweep survived a native null: %s", encode(t, input)) + } + }) + t.Run("max_trials null removes only max_trials", func(t *testing.T) { + input := lower(t, withParameters(withKVRange(defaultedSpec()), `{"sweep":{"max_trials":null,"seed":3}}`)) + if nested(t, input, "sweep", "max_trials") != nil || nested(t, input, "sweep", "seed") != 3.0 { + t.Fatalf("unexpected sweep: %s", encode(t, input)) + } + }) + t.Run("a native max_trials replaces the derived value", func(t *testing.T) { + input := lower(t, withParameters(withKVRange(defaultedSpec()), `{"sweep":{"max_trials":7}}`)) + if nested(t, input, "sweep", "max_trials") != 7.0 { + t.Fatalf("unexpected sweep: %s", encode(t, input)) + } + }) + t.Run("nothing is derived from values only parameters introduce", func(t *testing.T) { + input := lower(t, withParameters(defaultedSpec(), `{"workload":{"kv_load_ratio":[0,1]},"goal":{"sla":{"ttft_ms":500}}}`)) + if nested(t, input, "sweep") != nil || nested(t, input, "goal", "strict_sla") != nil { + t.Fatalf("a field was derived from native-only values: %s", encode(t, input)) + } + }) +} + +func TestUnderlayParetoObjective(t *testing.T) { + spec := defaultedSpec() + spec.Objective.Mode = v1beta2.ObjectiveModePareto + spec.Objective.Metric = nil + spec.Objective.Metrics = []v1beta2.ObjectiveMetric{"ThroughputPerGPU", "ThroughputPerUser", "Unknown"} + input := underlay(t, spec) + if nested(t, input, "goal", "target") != TargetPareto { + t.Fatalf("not a pareto goal: %v", input["goal"]) + } + objectives, _, _ := unstructured.NestedSlice(input, "goal", "pareto_objectives") + got, _ := json.Marshal(objectives) + if string(got) != `["throughput_per_gpu","throughput_per_user","unknown"]` { + t.Fatalf("pareto_objectives = %s", got) + } +} + +func TestUnderlayOmitsUnsetTypedFields(t *testing.T) { + spec := &v1beta2.DynamoGraphDeploymentRequestSpec{Model: &v1beta2.ModelSpec{Name: "m"}} + if got := encode(t, underlay(t, spec)); got != `{"search_space":{"model_name":"m"}}` { + t.Fatalf("got %s", got) + } + spec.Hardware = &v1beta2.HardwareSpec{GPU: &v1beta2.GPUHardwareSpec{SKU: ptrTo("")}} + if got := encode(t, underlay(t, spec)); got != `{"search_space":{"model_name":"m"}}` { + t.Fatalf("an empty SKU must not lower: %s", got) + } +} + +func TestUnderlayKVLoadRatioShapes(t *testing.T) { + spec := defaultedSpec() + spec.Workload.Static.KVLoadRatio = v1beta2.KVLoadRatio{0.25} + if nested(t, underlay(t, spec), "workload", "kv_load_ratio") != 0.25 { + t.Fatal("a single ratio lowers to a scalar") + } + spec.Workload.Static.KVLoadRatio = v1beta2.KVLoadRatio{0.1, 0.2, 0.3} + if ratios, _ := nested(t, underlay(t, spec), "workload", "kv_load_ratio").([]any); len(ratios) != 3 { + t.Fatal("longer lists lower unchanged so validation can report them") + } +} + +func TestParametersMustBeAnObject(t *testing.T) { + for _, parameters := range []string{`[1]`, `"x"`, `3`, `{`} { + if _, err := Lower(withParameters(defaultedSpec(), parameters)); err == nil { + t.Errorf("Lower accepted parameters %s", parameters) + } + } + for _, parameters := range []string{``, `null`, `{}`} { + if _, err := Lower(withParameters(defaultedSpec(), parameters)); err != nil { + t.Errorf("Lower rejected parameters %q: %v", parameters, err) + } + } +} + +func TestHasPath(t *testing.T) { + parameters := map[string]any{ + "goal": map[string]any{"target": nil, "sla": map[string]any{"ttft_ms": 1.0}}, + "workload": map[string]any{"isl": 8.0}, + } + for path, want := range map[string]bool{ + "goal.target": true, // a null still names the key + "goal.sla": true, + "goal.sla.ttft_ms": true, + "goal.sla.itl_ms": false, + "workload.isl": true, + "workload.osl": false, + "sweep.max_trials": false, + "goal.target.deeper": false, + "workload.isl.deeper": false, + } { + if got := has(parameters, strings.Split(path, ".")...); got != want { + t.Errorf("has(%s) = %v, want %v", path, got, want) + } + } + if has(nil, "goal") { + t.Error("nothing is named in nil parameters") + } +} + +func TestNativeMetricNames(t *testing.T) { + for metric, want := range map[v1beta2.ObjectiveMetric]string{ + "Throughput": "throughput", + "ThroughputPerGPU": "throughput_per_gpu", + "ThroughputPerUser": "throughput_per_user", + "TTFT": "ttft", + "E2ELatency": "e2e_latency", + "Goodput": "goodput", + "GoodputPerGPU": "goodput_per_gpu", + } { + if got := nativeMetric(metric); got != want { + t.Errorf("nativeMetric(%s) = %q, want %q", metric, got, want) + } + } +} diff --git a/deploy/operator/internal/controller/aisimulate/paths.go b/deploy/operator/internal/controller/aisimulate/paths.go new file mode 100644 index 000000000000..6992859b49e0 --- /dev/null +++ b/deploy/operator/internal/controller/aisimulate/paths.go @@ -0,0 +1,63 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package aisimulate + +import "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + +// Native input paths, in the snake_case spelling of the upstream AISimulate contract. +// They are read-only: callers pass them to the unstructured helpers and must not modify them. +var ( + SearchSpaceModelName = []string{"search_space", "model_name"} + SearchSpaceBackend = []string{"search_space", "backend"} + SearchSpaceHardwareSKU = []string{"search_space", "hardware_sku"} + SearchSpaceGPUBudget = []string{"search_space", "gpu_budget"} + + Workload = []string{"workload"} + WorkloadISL = []string{"workload", "isl"} + WorkloadOSL = []string{"workload", "osl"} + WorkloadNumRequestRatio = []string{"workload", "num_request_ratio"} + WorkloadRequestRate = []string{"workload", "request_rate"} + WorkloadConcurrency = []string{"workload", "concurrency"} + WorkloadKVLoadRatio = []string{"workload", "kv_load_ratio"} + WorkloadRequestCount = []string{"workload", "request_count"} + WorkloadTracePath = []string{"workload", "trace_path"} + WorkloadTracePaths = []string{"workload", "trace_paths"} + + Goal = []string{"goal"} + GoalTarget = []string{"goal", "target"} + GoalParetoObjectives = []string{"goal", "pareto_objectives"} + GoalSLA = []string{"goal", "sla"} + GoalSLATTFTMs = []string{"goal", "sla", "ttft_ms"} + GoalSLAITLMs = []string{"goal", "sla", "itl_ms"} + GoalSLAE2EMs = []string{"goal", "sla", "e2e_ms"} + GoalStrictSLA = []string{"goal", "strict_sla"} + GoalMinGoodputRPS = []string{"goal", "min_goodput_rps"} + + SweepMaxTrials = []string{"sweep", "max_trials"} +) + +// TargetPareto is the goal.target of a multi-objective search. +const TargetPareto = "pareto" + +// has reports whether obj contains the path, even with a null value. Applied to +// spec.aisimulate.parameters it tells whether the caller named a key, including to delete +// it. +func has(obj map[string]any, path ...string) bool { + _, found, _ := unstructured.NestedFieldNoCopy(obj, path...) + return found +} diff --git a/deploy/operator/internal/controller/dynamographdeploymentrequest_controller.go b/deploy/operator/internal/controller/dynamographdeploymentrequest_controller.go index 67f35c3a1bc7..1d24fd6060a8 100644 --- a/deploy/operator/internal/controller/dynamographdeploymentrequest_controller.go +++ b/deploy/operator/internal/controller/dynamographdeploymentrequest_controller.go @@ -26,7 +26,6 @@ import ( "io" "reflect" "sort" - "strings" "text/template" "time" @@ -1503,52 +1502,11 @@ func (r *DynamoGraphDeploymentRequestReconciler) validateGPUHardwareInfo(ctx con "or set spec.hardware.{gpuSku,vramMb,numGpusPerNode} explicitly.") } -// GetGPUDiscoveryFailureReason classifies a GPU discovery error and -// returns a stable, actionable reason string suitable for structured logging. -// -// The classification is based on known error message patterns produced during: -// - DCGM exporter pod discovery -// - Helm-based GPU operator and DCGM discovery -// - Metrics scraping -// - Prometheus parsing -// -// If the error does not match any known category, "unknown" is returned. +// GetGPUDiscoveryFailureReason classifies a GPU discovery error into a stable reason string +// for structured logging. The classification lives in the gpu package so the v1beta2 +// admission path reports the same reasons. func GetGPUDiscoveryFailureReason(err error) string { - const unknownReason = "unknown" - - if err == nil { - return unknownReason - } - errMsg := strings.ToLower(err.Error()) - - switch { - case strings.Contains(errMsg, "list pods"): - return "failed to list DCGM exporter pods (RBAC/cluster connectivity issue)" - case strings.Contains(errMsg, "gpu operator is not installed"): - return "GPU Operator not installed in expected namespace" - case strings.Contains(errMsg, "helm init failed"): - return "failed to initialize Helm client (RBAC, kubeconfig, or Helm driver issue)" - case strings.Contains(errMsg, "timeout waiting for dcgm exporter pods"): - return "timeout while waiting for DCGM exporter pods to become ready" - case strings.Contains(errMsg, "http get"): - return "failed to reach DCGM metrics endpoint on pod (network/port issue)" - case strings.Contains(errMsg, "metrics endpoint") && - strings.Contains(errMsg, "status"): - return "DCGM pod metrics endpoint returned non-200 status" - case strings.Contains(errMsg, "parse prometheus metrics"): - return "failed to parse dcgm Prometheus metrics (invalid format)" - case strings.Contains(errMsg, "no gpus detected"): - return "no GPUs detected in dcgm metrics (GPU model or metrics missing)" - case strings.Contains(errMsg, "dcgm is not enabled in the GPU Operator"): - return "DCGM is not enabled in the GPU Operator (check GPU Operator configuration and permissions)" - case strings.Contains(errMsg, "failed to scrape any dcgm exporter pod"): - return "failed to scrape any dcgm exporter pod (check DCGM exporter pod status and network connectivity)" - case strings.Contains(errMsg, "no gpu metrics could be parsed from any dcgm pod"): - return "no GPU metrics could be parsed from any DCGM pod (check DCGM exporter pod status and network connectivity)" - case strings.Contains(errMsg, "failed to create helm path"): - return "failed to initialize Helm client (RBAC, kubeconfig, or Helm driver issue)" - } - return unknownReason + return gpu.DiscoveryFailureReason(err) } // createProfilingJob creates a Kubernetes Job for profiling using SyncResource. @@ -1999,41 +1957,21 @@ func (r *DynamoGraphDeploymentRequestReconciler) discoverHardwareForEnrichment(c return nil, nil } - var dcgmErr error - if r.GPUDiscovery != nil { - discoveredInfo, err := r.GPUDiscovery.DiscoverGPUsFromDCGMFiltered(ctx, reader, r.GPUDiscoveryCache, hw.GPUSKU) - if err == nil { - return discoveredInfo, nil - } - dcgmErr = err - - reason := GetGPUDiscoveryFailureReason(err) - logger.Info("DCGM discovery failed, falling back to node-label discovery", - "reason", reason, "error", err.Error()) + discoverer := &gpu.Discoverer{ + Reader: reader, + DCGM: r.GPUDiscovery, + Cache: r.GPUDiscoveryCache, + NodeLabels: r.gpuDiscoveryEnabled, } - - if !r.gpuDiscoveryEnabled() { + info, err := discoverer.Discover(ctx, hw.GPUSKU) + if err != nil { if discoveryRequired { - if dcgmErr != nil { - return nil, fmt.Errorf("auto-discovery failed: %w", dcgmErr) - } - return nil, fmt.Errorf("auto-discovery failed: node-label discovery is disabled") + return nil, err } - logger.Info("Optional hardware metadata discovery skipped because node-label discovery is disabled") + logger.Info("Optional hardware metadata discovery unavailable; leaving unset fields unchanged", "error", err.Error()) return nil, nil } - - discoveredInfo, err := gpu.DiscoverGPUsFiltered(ctx, reader, hw.GPUSKU) - if err == nil { - return discoveredInfo, nil - } - - logger.Info("Node-label discovery also failed", "error", err.Error()) - if discoveryRequired { - return nil, fmt.Errorf("auto-discovery failed: %w", err) - } - logger.Info("Optional hardware metadata discovery unavailable; leaving unset fields unchanged") - return nil, nil + return info, nil } // extractModelCachePVCConfig reads model cache PVC settings from the typed v1beta1 spec. diff --git a/deploy/operator/internal/controller/suite_envtest_test.go b/deploy/operator/internal/controller/suite_envtest_test.go index 45651e30d0f0..88ca0c68076b 100644 --- a/deploy/operator/internal/controller/suite_envtest_test.go +++ b/deploy/operator/internal/controller/suite_envtest_test.go @@ -9,10 +9,12 @@ package controller import ( + "context" "os" "testing" "github.com/ai-dynamo/dynamo/deploy/operator/internal/testing/operatorenv" + webhookdefaulting "github.com/ai-dynamo/dynamo/deploy/operator/internal/webhook/defaulting" webhooksetup "github.com/ai-dynamo/dynamo/deploy/operator/internal/webhook/setup" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -24,6 +26,16 @@ import ( const admissionBypassUsername = "operatorenv-controller-admission-bypass" +// The controller specs create native v1beta2 Requests without an image or hardware. Admission +// completes them from the operator's configuration, as it does in production. +const controllerAISimulateImage = "registry.example/aisimulate:1.2.0" + +type fixedHardwareDiscoverer struct{} + +func (fixedHardwareDiscoverer) Discover(context.Context, string) (webhookdefaulting.DiscoveredHardware, error) { + return webhookdefaulting.DiscoveredHardware{SKU: "h200_sxm", GPUs: 8}, nil +} + var ( sharedEnv = operatorenv.New(operatorenv.Options{ Admission: operatorenv.AdmissionWebhooks{ @@ -44,6 +56,10 @@ func setupProductionWebhooks(mgr ctrl.Manager, opts operatorenv.WebhookSetupOpti RuntimeConfig: opts.RuntimeConfig, OperatorVersion: opts.OperatorVersion, OperatorPrincipal: opts.OperatorPrincipal, + DGDRV1beta2: webhookdefaulting.DGDRV1beta2Options{ + AISimulateImage: controllerAISimulateImage, + Hardware: fixedHardwareDiscoverer{}, + }, }) } diff --git a/deploy/operator/internal/dynamo/component_common.go b/deploy/operator/internal/dynamo/component_common.go index 404f07a544a9..ca4e1f2e9506 100644 --- a/deploy/operator/internal/dynamo/component_common.go +++ b/deploy/operator/internal/dynamo/component_common.go @@ -10,7 +10,7 @@ import ( "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" commonconsts "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" controller_common "github.com/ai-dynamo/dynamo/deploy/operator/internal/controller_common" - "github.com/ai-dynamo/dynamo/deploy/operator/internal/runtimeversion" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/versionoverride" corev1 "k8s.io/api/core/v1" "k8s.io/utils/ptr" ) @@ -70,7 +70,7 @@ type ComponentContext struct { Infrastructure configv1alpha1.InfrastructureConfiguration EPPConfig *v1beta1.EPPConfig WorkerHashSuffix string - RuntimeVersion *runtimeversion.Version + RuntimeVersion *versionoverride.Version } func (b *BaseComponentDefaults) GetBaseContainer(context ComponentContext) (corev1.Container, error) { diff --git a/deploy/operator/internal/dynamo/component_frontend_test.go b/deploy/operator/internal/dynamo/component_frontend_test.go index ac178e3b8be0..75129dd10f5d 100644 --- a/deploy/operator/internal/dynamo/component_frontend_test.go +++ b/deploy/operator/internal/dynamo/component_frontend_test.go @@ -11,7 +11,7 @@ import ( configv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/config/v1alpha1" "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" commonconsts "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" - "github.com/ai-dynamo/dynamo/deploy/operator/internal/runtimeversion" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/versionoverride" "github.com/stretchr/testify/require" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/util/intstr" @@ -21,13 +21,13 @@ import ( func TestFrontendDefaultsStartupProbeVersionGate(t *testing.T) { tests := []struct { name string - runtimeVersion *runtimeversion.Version + runtimeVersion *versionoverride.Version wantEnabled bool }{ {name: "unknown legacy runtime"}, - {name: "older runtime", runtimeVersion: &runtimeversion.Version{Major: 1, Minor: 5, Patch: 9}}, - {name: "minimum supported runtime", runtimeVersion: &runtimeversion.Version{Major: 1, Minor: 6}, wantEnabled: true}, - {name: "newer runtime", runtimeVersion: &runtimeversion.Version{Major: 2}, wantEnabled: true}, + {name: "older runtime", runtimeVersion: &versionoverride.Version{Major: 1, Minor: 5, Patch: 9}}, + {name: "minimum supported runtime", runtimeVersion: &versionoverride.Version{Major: 1, Minor: 6}, wantEnabled: true}, + {name: "newer runtime", runtimeVersion: &versionoverride.Version{Major: 2}, wantEnabled: true}, } for _, tt := range tests { diff --git a/deploy/operator/internal/dynamo/component_namespace_test.go b/deploy/operator/internal/dynamo/component_namespace_test.go index 5919964c0a86..44aebfefcb3f 100644 --- a/deploy/operator/internal/dynamo/component_namespace_test.go +++ b/deploy/operator/internal/dynamo/component_namespace_test.go @@ -11,7 +11,7 @@ import ( configv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/config/v1alpha1" v1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" commonconsts "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" - "github.com/ai-dynamo/dynamo/deploy/operator/internal/runtimeversion" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/versionoverride" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" corev1 "k8s.io/api/core/v1" @@ -23,13 +23,13 @@ func TestComponentNamespacePrefixRuntimeCompatibility(t *testing.T) { t.Log("render frontend and EPP defaults across the strict-discovery release boundary") versions := []struct { name string - version *runtimeversion.Version + version *versionoverride.Version strict bool }{ {name: "unknown runtime"}, - {name: "older runtime", version: &runtimeversion.Version{Major: 1, Minor: 5, Patch: 9}}, - {name: "supported runtime", version: &runtimeversion.Version{Major: 1, Minor: 6, Patch: 0}, strict: true}, - {name: "newer runtime", version: &runtimeversion.Version{Major: 1, Minor: 7, Patch: 0}, strict: true}, + {name: "older runtime", version: &versionoverride.Version{Major: 1, Minor: 5, Patch: 9}}, + {name: "supported runtime", version: &versionoverride.Version{Major: 1, Minor: 6, Patch: 0}, strict: true}, + {name: "newer runtime", version: &versionoverride.Version{Major: 1, Minor: 7, Patch: 0}, strict: true}, } for _, componentType := range []string{commonconsts.ComponentTypeFrontend, commonconsts.ComponentTypeEPP} { @@ -60,7 +60,7 @@ func TestLegacyEPPDoesNotEnableStrictNamespacePrefix(t *testing.T) { container, err := NewEPPDefaults().GetBaseContainer(ComponentContext{ DynamoNamespace: "default-foo", ComponentType: commonconsts.ComponentTypeEPP, - RuntimeVersion: &runtimeversion.Version{Major: 1, Minor: 6, Patch: 0}, + RuntimeVersion: &versionoverride.Version{Major: 1, Minor: 6, Patch: 0}, EPPConfig: &v1beta1.EPPConfig{}, }) require.NoError(t, err) diff --git a/deploy/operator/internal/dynamo/component_worker_test.go b/deploy/operator/internal/dynamo/component_worker_test.go index aa39447c5b40..2c399d60724e 100644 --- a/deploy/operator/internal/dynamo/component_worker_test.go +++ b/deploy/operator/internal/dynamo/component_worker_test.go @@ -9,14 +9,14 @@ import ( "testing" commonconsts "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" - "github.com/ai-dynamo/dynamo/deploy/operator/internal/runtimeversion" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/versionoverride" "github.com/stretchr/testify/require" ) func TestWorkerDefaultsCanaryHealthCheckVersionGate(t *testing.T) { tests := []struct { name string - runtimeVersion *runtimeversion.Version + runtimeVersion *versionoverride.Version wantEnabled string }{ { @@ -25,17 +25,17 @@ func TestWorkerDefaultsCanaryHealthCheckVersionGate(t *testing.T) { }, { name: "older runtime", - runtimeVersion: &runtimeversion.Version{Major: 1, Minor: 4, Patch: 9}, + runtimeVersion: &versionoverride.Version{Major: 1, Minor: 4, Patch: 9}, wantEnabled: "false", }, { name: "minimum supported runtime", - runtimeVersion: &runtimeversion.Version{Major: 1, Minor: 5, Patch: 0}, + runtimeVersion: &versionoverride.Version{Major: 1, Minor: 5, Patch: 0}, wantEnabled: "true", }, { name: "newer runtime", - runtimeVersion: &runtimeversion.Version{Major: 2, Minor: 0, Patch: 0}, + runtimeVersion: &versionoverride.Version{Major: 2, Minor: 0, Patch: 0}, wantEnabled: "true", }, } @@ -64,7 +64,7 @@ func TestWorkerDefaultsCanaryHealthCheckVersionGate(t *testing.T) { func TestWorkerDefaultsLivenessFailureThresholdVersionGate(t *testing.T) { tests := []struct { name string - runtimeVersion *runtimeversion.Version + runtimeVersion *versionoverride.Version wantFailureCount int32 }{ { @@ -73,17 +73,17 @@ func TestWorkerDefaultsLivenessFailureThresholdVersionGate(t *testing.T) { }, { name: "older runtime", - runtimeVersion: &runtimeversion.Version{Major: 1, Minor: 4, Patch: 9}, + runtimeVersion: &versionoverride.Version{Major: 1, Minor: 4, Patch: 9}, wantFailureCount: 1, }, { name: "minimum supported runtime", - runtimeVersion: &runtimeversion.Version{Major: 1, Minor: 5, Patch: 0}, + runtimeVersion: &versionoverride.Version{Major: 1, Minor: 5, Patch: 0}, wantFailureCount: 3, }, { name: "newer runtime", - runtimeVersion: &runtimeversion.Version{Major: 2, Minor: 0, Patch: 0}, + runtimeVersion: &versionoverride.Version{Major: 2, Minor: 0, Patch: 0}, wantFailureCount: 3, }, } diff --git a/deploy/operator/internal/dynamo/graph.go b/deploy/operator/internal/dynamo/graph.go index ef40a40e98ac..e13bcd3a006a 100644 --- a/deploy/operator/internal/dynamo/graph.go +++ b/deploy/operator/internal/dynamo/graph.go @@ -44,7 +44,7 @@ import ( "github.com/ai-dynamo/dynamo/deploy/operator/internal/features" "github.com/ai-dynamo/dynamo/deploy/operator/internal/features/compatibility" gms "github.com/ai-dynamo/dynamo/deploy/operator/internal/gms" - "github.com/ai-dynamo/dynamo/deploy/operator/internal/runtimeversion" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/versionoverride" grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" "github.com/imdario/mergo" "google.golang.org/protobuf/types/known/wrapperspb" @@ -2176,8 +2176,8 @@ func mergeFrontendSidecarDefaults(podSpec *corev1.PodSpec, sidecarName string, p } // Resolve the frontend image independently of the component runtime. - var resolvedRuntimeVersion *runtimeversion.Version - if version, err := runtimeversion.ParseImageVersion(podSpec.Containers[i].Image); err == nil { + var resolvedRuntimeVersion *versionoverride.Version + if version, err := versionoverride.ParseImageVersion(podSpec.Containers[i].Image); err == nil { resolvedRuntimeVersion = &version } @@ -2270,8 +2270,8 @@ func generateComponentContext(component *v1beta1.DynamoComponentDeploymentShared if runtime := GetDynamoContainer(component); runtime != nil { image = runtime.Image } - var resolvedRuntimeVersion *runtimeversion.Version - if version, err := runtimeversion.Resolve(image, component.RuntimeVersionOverride); err == nil { + var resolvedRuntimeVersion *versionoverride.Version + if version, err := versionoverride.Resolve(image, component.RuntimeVersionOverride); err == nil { resolvedRuntimeVersion = &version } else if component.RuntimeVersionOverride != "" { return ComponentContext{}, fmt.Errorf("resolve runtime version override: %w", err) diff --git a/deploy/operator/internal/dynamo/hash.go b/deploy/operator/internal/dynamo/hash.go index 4dd87b283c00..de2be302a719 100644 --- a/deploy/operator/internal/dynamo/hash.go +++ b/deploy/operator/internal/dynamo/hash.go @@ -26,7 +26,7 @@ import ( "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" "github.com/ai-dynamo/dynamo/deploy/operator/internal/provideroverride" - "github.com/ai-dynamo/dynamo/deploy/operator/internal/runtimeversion" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/versionoverride" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" "k8s.io/utils/ptr" ) @@ -34,7 +34,7 @@ import ( const dgdWorkerHashPlaceholderValue = "worker-hash-placeholder" // Preserve hashes for runtimes released before version hashing. -var minimumHashedRuntimeVersion = runtimeversion.Version{Major: 1, Minor: 5, Patch: 0} +var minimumHashedRuntimeVersion = versionoverride.Version{Major: 1, Minor: 5, Patch: 0} // ComputeDGDWorkersSpecHash computes the v2 worker hash from the worker DCDs // generated by the same DGD-to-DCD path used by reconciliation. The active @@ -223,7 +223,7 @@ func resolvedRuntimeVersionForHash(component *v1beta1.DynamoComponentDeploymentS if runtime := GetDynamoContainer(component); runtime != nil { image = runtime.Image } - version, err := runtimeversion.Resolve(image, component.RuntimeVersionOverride) + version, err := versionoverride.Resolve(image, component.RuntimeVersionOverride) if err != nil || version.Compare(minimumHashedRuntimeVersion) < 0 { return "" } diff --git a/deploy/operator/internal/features/runtime/gate.go b/deploy/operator/internal/features/runtime/gate.go index 805099b09e95..66a3aedab213 100644 --- a/deploy/operator/internal/features/runtime/gate.go +++ b/deploy/operator/internal/features/runtime/gate.go @@ -5,16 +5,16 @@ package runtime -import "github.com/ai-dynamo/dynamo/deploy/operator/internal/runtimeversion" +import "github.com/ai-dynamo/dynamo/deploy/operator/internal/versionoverride" // Gate controls runtime-version-dependent operator behavior. type Gate struct { Name string - MinRuntimeVersion runtimeversion.Version + MinRuntimeVersion versionoverride.Version } // Enabled reports whether a known runtime version meets the feature threshold. -func (g Gate) Enabled(version *runtimeversion.Version) bool { +func (g Gate) Enabled(version *versionoverride.Version) bool { if version == nil { return false } diff --git a/deploy/operator/internal/features/runtime/gate_test.go b/deploy/operator/internal/features/runtime/gate_test.go index c3919a34b988..b9ea9db53738 100644 --- a/deploy/operator/internal/features/runtime/gate_test.go +++ b/deploy/operator/internal/features/runtime/gate_test.go @@ -8,20 +8,20 @@ package runtime import ( "testing" - "github.com/ai-dynamo/dynamo/deploy/operator/internal/runtimeversion" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/versionoverride" ) func TestGateEnabled(t *testing.T) { t.Log("define a feature introduced by Dynamo runtime 1.4.0") gate := Gate{ Name: "TestFeature", - MinRuntimeVersion: runtimeversion.Version{Major: 1, Minor: 4, Patch: 0}, + MinRuntimeVersion: versionoverride.Version{Major: 1, Minor: 4, Patch: 0}, } t.Log("define versions below, at, and above the feature threshold") tests := []struct { name string - version *runtimeversion.Version + version *versionoverride.Version want bool }{ { @@ -29,16 +29,16 @@ func TestGateEnabled(t *testing.T) { }, { name: "older runtime", - version: &runtimeversion.Version{Major: 1, Minor: 3, Patch: 9}, + version: &versionoverride.Version{Major: 1, Minor: 3, Patch: 9}, }, { name: "minimum supported runtime", - version: &runtimeversion.Version{Major: 1, Minor: 4, Patch: 0}, + version: &versionoverride.Version{Major: 1, Minor: 4, Patch: 0}, want: true, }, { name: "newer runtime", - version: &runtimeversion.Version{Major: 2, Minor: 0, Patch: 0}, + version: &versionoverride.Version{Major: 2, Minor: 0, Patch: 0}, want: true, }, } diff --git a/deploy/operator/internal/features/runtime/gates.go b/deploy/operator/internal/features/runtime/gates.go index b77533226721..ee21c27072c2 100644 --- a/deploy/operator/internal/features/runtime/gates.go +++ b/deploy/operator/internal/features/runtime/gates.go @@ -5,19 +5,19 @@ package runtime -import "github.com/ai-dynamo/dynamo/deploy/operator/internal/runtimeversion" +import "github.com/ai-dynamo/dynamo/deploy/operator/internal/versionoverride" var ( // StrictNamespacePrefix gates operator-managed discovery isolation. StrictNamespacePrefix = Gate{ Name: "StrictNamespacePrefix", - MinRuntimeVersion: runtimeversion.Version{Major: 1, Minor: 6, Patch: 0}, + MinRuntimeVersion: versionoverride.Version{Major: 1, Minor: 6, Patch: 0}, } // FrontendStartupProbe allows the sequential NATS and etcd startup connection windows. FrontendStartupProbe = Gate{ Name: "FrontendStartupProbe", - MinRuntimeVersion: runtimeversion.Version{Major: 1, Minor: 6, Patch: 0}, + MinRuntimeVersion: versionoverride.Version{Major: 1, Minor: 6, Patch: 0}, } // CanaryHealthChecks gates the canary health-check rendering defaults. @@ -26,7 +26,7 @@ var ( // change an existing worker generation. CanaryHealthChecks = Gate{ Name: "CanaryHealthChecks", - MinRuntimeVersion: runtimeversion.Version{Major: 1, Minor: 5, Patch: 0}, + MinRuntimeVersion: versionoverride.Version{Major: 1, Minor: 5, Patch: 0}, } // IncreasedWorkerFailureThreshold gates the worker liveness default. @@ -35,7 +35,7 @@ var ( // change an existing worker generation. IncreasedWorkerFailureThreshold = Gate{ Name: "IncreasedWorkerFailureThreshold", - MinRuntimeVersion: runtimeversion.Version{Major: 1, Minor: 5, Patch: 0}, + MinRuntimeVersion: versionoverride.Version{Major: 1, Minor: 5, Patch: 0}, } // NativeRustEPP gates the native Rust EPP requirement: runtime versions at @@ -44,7 +44,7 @@ var ( // 1.5.0, the first release shipping the native Rust EPP image. NativeRustEPP = Gate{ Name: "NativeRustEPP", - MinRuntimeVersion: runtimeversion.Version{Major: 1, Minor: 5, Patch: 0}, + MinRuntimeVersion: versionoverride.Version{Major: 1, Minor: 5, Patch: 0}, } // PlannerDGDComponentStatus gates Planner discovery of model, runtime, @@ -54,6 +54,6 @@ var ( // PodTemplates. PlannerDGDComponentStatus = Gate{ Name: "PlannerDGDComponentStatus", - MinRuntimeVersion: runtimeversion.Version{Major: 1, Minor: 6, Patch: 0}, + MinRuntimeVersion: versionoverride.Version{Major: 1, Minor: 6, Patch: 0}, } ) diff --git a/deploy/operator/internal/gpu/hardware_discoverer.go b/deploy/operator/internal/gpu/hardware_discoverer.go new file mode 100644 index 000000000000..6f060a22e291 --- /dev/null +++ b/deploy/operator/internal/gpu/hardware_discoverer.go @@ -0,0 +1,140 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package gpu + +import ( + "context" + "errors" + "fmt" + "strings" + + nvidiacomv1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/log" +) + +// Discoverer finds the cluster's GPUs the way the DGDR controller does: DCGM first, then +// node labels when node-label discovery is enabled. It is a plain value so callers other +// than the v1beta1 reconciler (the v1beta2 admission webhook) share the same order and the +// same error wording. +type Discoverer struct { + Reader client.Reader + // DCGM scrapes the DCGM exporter. Nil skips it. + DCGM *GPUDiscovery + Cache *GPUDiscoveryCache + // NodeLabels reports whether the node-label fallback is enabled (the GPUDiscovery feature + // gate). It is a func so the gate is only read when DCGM did not answer, as the v1beta1 + // reconciler always did. Nil means disabled. + NodeLabels func() bool +} + +// Discover returns the discovered GPUs, restricted to filterSKU when it is set. An error +// means no backend produced a result. +func (d *Discoverer) Discover(ctx context.Context, filterSKU nvidiacomv1beta1.GPUSKUType) (*GPUInfo, error) { + logger := log.FromContext(ctx) + if d == nil || d.Reader == nil { + return nil, errors.New("auto-discovery failed: APIReader is not configured") + } + + var dcgmErr error + if d.DCGM != nil { + info, err := d.DCGM.DiscoverGPUsFromDCGMFiltered(ctx, d.Reader, d.Cache, filterSKU) + if err == nil { + return info, nil + } + dcgmErr = err + logger.Info("DCGM discovery failed, falling back to node-label discovery", + "reason", DiscoveryFailureReason(err), "error", err.Error()) + } + + if d.NodeLabels == nil || !d.NodeLabels() { + if dcgmErr != nil { + return nil, fmt.Errorf("auto-discovery failed: %w", dcgmErr) + } + return nil, errors.New("auto-discovery failed: node-label discovery is disabled") + } + + info, err := DiscoverGPUsFiltered(ctx, d.Reader, filterSKU) + if err != nil { + logger.Info("Node-label discovery also failed", "error", err.Error()) + return nil, fmt.Errorf("auto-discovery failed: %w", err) + } + return info, nil +} + +// KnownSKU returns the hardware catalog name of the discovered GPU, or empty when the model +// is not one the catalog knows. Unlike the v1beta1 reconciler it never falls back to the raw +// model string: a v1beta2 SKU is an AISimulate catalog name, and an unknown name would only +// fail later inside the search. +func KnownSKU(info *GPUInfo) nvidiacomv1beta1.GPUSKUType { + if info == nil { + return "" + } + if info.System != "" { + return info.System + } + return InferHardwareSystem(info.Model) +} + +// DiscoveryFailureReason classifies a GPU discovery error and +// returns a stable, actionable reason string suitable for structured logging. +// +// The classification is based on known error message patterns produced during: +// - DCGM exporter pod discovery +// - Helm-based GPU operator and DCGM discovery +// - Metrics scraping +// - Prometheus parsing +// +// If the error does not match any known category, "unknown" is returned. +func DiscoveryFailureReason(err error) string { + const unknownReason = "unknown" + + if err == nil { + return unknownReason + } + errMsg := strings.ToLower(err.Error()) + + switch { + case strings.Contains(errMsg, "list pods"): + return "failed to list DCGM exporter pods (RBAC/cluster connectivity issue)" + case strings.Contains(errMsg, "gpu operator is not installed"): + return "GPU Operator not installed in expected namespace" + case strings.Contains(errMsg, "helm init failed"): + return "failed to initialize Helm client (RBAC, kubeconfig, or Helm driver issue)" + case strings.Contains(errMsg, "timeout waiting for dcgm exporter pods"): + return "timeout while waiting for DCGM exporter pods to become ready" + case strings.Contains(errMsg, "http get"): + return "failed to reach DCGM metrics endpoint on pod (network/port issue)" + case strings.Contains(errMsg, "metrics endpoint") && + strings.Contains(errMsg, "status"): + return "DCGM pod metrics endpoint returned non-200 status" + case strings.Contains(errMsg, "parse prometheus metrics"): + return "failed to parse dcgm Prometheus metrics (invalid format)" + case strings.Contains(errMsg, "no gpus detected"): + return "no GPUs detected in dcgm metrics (GPU model or metrics missing)" + case strings.Contains(errMsg, "dcgm is not enabled in the GPU Operator"): + return "DCGM is not enabled in the GPU Operator (check GPU Operator configuration and permissions)" + case strings.Contains(errMsg, "failed to scrape any dcgm exporter pod"): + return "failed to scrape any dcgm exporter pod (check DCGM exporter pod status and network connectivity)" + case strings.Contains(errMsg, "no gpu metrics could be parsed from any dcgm pod"): + return "no GPU metrics could be parsed from any DCGM pod (check DCGM exporter pod status and network connectivity)" + case strings.Contains(errMsg, "failed to create helm path"): + return "failed to initialize Helm client (RBAC, kubeconfig, or Helm driver issue)" + } + return unknownReason +} diff --git a/deploy/operator/internal/gpu/hardware_discoverer_test.go b/deploy/operator/internal/gpu/hardware_discoverer_test.go new file mode 100644 index 000000000000..ef0493c42197 --- /dev/null +++ b/deploy/operator/internal/gpu/hardware_discoverer_test.go @@ -0,0 +1,90 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package gpu + +import ( + "context" + "errors" + "strings" + "testing" + + nvidiacomv1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" + "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +func TestDiscovererWithoutReader(t *testing.T) { + var nilDiscoverer *Discoverer + for _, d := range []*Discoverer{nilDiscoverer, {}} { + if _, err := d.Discover(context.Background(), ""); err == nil || !strings.Contains(err.Error(), "APIReader is not configured") { + t.Fatalf("got %v", err) + } + } +} + +func TestDiscovererWithNoBackend(t *testing.T) { + d := &Discoverer{Reader: fake.NewClientBuilder().Build()} + if _, err := d.Discover(context.Background(), ""); err == nil || !strings.Contains(err.Error(), "node-label discovery is disabled") { + t.Fatalf("got %v", err) + } +} + +func TestDiscovererNodeLabelsFindNothing(t *testing.T) { + d := &Discoverer{Reader: fake.NewClientBuilder().Build(), NodeLabels: func() bool { return true }} + if _, err := d.Discover(context.Background(), ""); err == nil || !strings.Contains(err.Error(), "auto-discovery failed") { + t.Fatalf("an empty cluster must be an error, got %v", err) + } +} + +func TestKnownSKU(t *testing.T) { + cases := []struct { + name string + info *GPUInfo + want nvidiacomv1beta1.GPUSKUType + }{ + {"nil", nil, ""}, + {"discovered system wins", &GPUInfo{System: "h100_sxm", Model: "NVIDIA H200"}, "h100_sxm"}, + {"inferred from the model", &GPUInfo{Model: "NVIDIA H200"}, "h200_sxm"}, + {"unknown model is not passed through", &GPUInfo{Model: "NVIDIA RTX 6000 Ada"}, ""}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := KnownSKU(tc.info); got != tc.want { + t.Fatalf("got %q want %q", got, tc.want) + } + }) + } +} + +func TestDiscoveryFailureReason(t *testing.T) { + cases := map[string]string{ + "list pods: forbidden": "failed to list DCGM exporter pods (RBAC/cluster connectivity issue)", + "GPU Operator is not installed": "GPU Operator not installed in expected namespace", + "http get failed": "failed to reach DCGM metrics endpoint on pod (network/port issue)", + "metrics endpoint returned status": "DCGM pod metrics endpoint returned non-200 status", + "no GPUs detected": "no GPUs detected in dcgm metrics (GPU model or metrics missing)", + "something else": "unknown", + } + for msg, want := range cases { + if got := DiscoveryFailureReason(errors.New(msg)); got != want { + t.Errorf("%q: got %q, want %q", msg, got, want) + } + } + if got := DiscoveryFailureReason(nil); got != "unknown" { + t.Errorf("nil: got %q", got) + } +} diff --git a/deploy/operator/internal/runtimeversion/runtimeversion.go b/deploy/operator/internal/versionoverride/versionoverride.go similarity index 99% rename from deploy/operator/internal/runtimeversion/runtimeversion.go rename to deploy/operator/internal/versionoverride/versionoverride.go index eab91c6c48d8..c400bce11af7 100644 --- a/deploy/operator/internal/runtimeversion/runtimeversion.go +++ b/deploy/operator/internal/versionoverride/versionoverride.go @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package runtimeversion +package versionoverride import ( "cmp" diff --git a/deploy/operator/internal/runtimeversion/runtimeversion_test.go b/deploy/operator/internal/versionoverride/versionoverride_test.go similarity index 99% rename from deploy/operator/internal/runtimeversion/runtimeversion_test.go rename to deploy/operator/internal/versionoverride/versionoverride_test.go index edc26679c343..8cccb8005eef 100644 --- a/deploy/operator/internal/runtimeversion/runtimeversion_test.go +++ b/deploy/operator/internal/versionoverride/versionoverride_test.go @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package runtimeversion +package versionoverride import "testing" diff --git a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_handler.go b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_handler.go index a828ad8f145b..eb6ef7dca92e 100644 --- a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_handler.go +++ b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_handler.go @@ -63,6 +63,9 @@ type DGDRDefaulter struct { // dynamo-planner: tag does not exist (pre-release charts: // rc, nightly); empty keeps the derived default. DefaultImage string + // v1beta2 configures the defaults of Requests submitted as v1beta2, which reach this + // handler as a v1beta1 object carrying the spec.v1beta2 envelope. + v1beta2 DGDRV1beta2Options } // NewDGDRDefaulter creates a new DGDRDefaulter with the given operator version @@ -71,6 +74,12 @@ func NewDGDRDefaulter(operatorVersion, defaultImage string) *DGDRDefaulter { return &DGDRDefaulter{OperatorVersion: operatorVersion, DefaultImage: defaultImage} } +// WithV1beta2 sets the defaults of v1beta2 Requests and returns the defaulter. +func (d *DGDRDefaulter) WithV1beta2(options DGDRV1beta2Options) *DGDRDefaulter { + d.v1beta2 = options + return d +} + // Default implements admission.CustomDefaulter. // Native API defaults apply on CREATE and UPDATE. If spec.image is not set on // CREATE, derives a default image from the operator version. @@ -92,7 +101,14 @@ func (d *DGDRDefaulter) Default(ctx context.Context, obj runtime.Object) error { return nil } - if dgdr.Spec.V1Beta2 != nil { + // The original request version decides what spec.v1beta2 means. A native v1beta2 Request + // is defaulted from its carrier on CREATE and UPDATE; only CREATE reads the cluster. + if internalwebhook.RequestUsesV1Beta2(req) && dgdr.Spec.V1Beta2 != nil { + return defaultDynamoGraphDeploymentRequestV1beta2(ctx, dgdr, req.Operation, d.v1beta2) + } + // A foreign-version view (a carrier written or updated through the wrong API version, or + // a v1beta1 payload written through v1beta2) is read-only; validation enforces that. + if !internalwebhook.RequestIsNative(req, dgdr.Spec.V1Beta2 != nil) { return nil } diff --git a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_routing_test.go b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_routing_test.go new file mode 100644 index 000000000000..7dbb6acd8950 --- /dev/null +++ b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_routing_test.go @@ -0,0 +1,129 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package defaulting + +import ( + "bytes" + "context" + "encoding/json" + "testing" + + nvidiacomv1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" + admissionv1 "k8s.io/api/admission/v1" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" +) + +// ctxForRequest builds the admission context of a request the API server converted to +// v1beta1. requestVersion is the version the client used; empty means no conversion. +func ctxForRequest(op admissionv1.Operation, requestVersion string) context.Context { + gvk := nvidiacomv1beta1.DynamoGraphDeploymentRequestGVK + req := admission.Request{AdmissionRequest: admissionv1.AdmissionRequest{ + Operation: op, + Kind: metav1.GroupVersionKind{Group: gvk.Group, Version: gvk.Version, Kind: gvk.Kind}, + }} + if requestVersion != "" { + req.RequestKind = &metav1.GroupVersionKind{Group: gvk.Group, Version: requestVersion, Kind: gvk.Kind} + } + return admission.NewContextWithRequest(context.Background(), req) +} + +const modelOnlyCarrier = `{"model":{"name":"Qwen/Qwen3-32B"}}` + +func carrierRequest() *nvidiacomv1beta1.DynamoGraphDeploymentRequest { + return &nvidiacomv1beta1.DynamoGraphDeploymentRequest{ + ObjectMeta: metav1.ObjectMeta{Name: "r", Namespace: "default"}, + Spec: nvidiacomv1beta1.DynamoGraphDeploymentRequestSpec{ + V1Beta2: &apiextensionsv1.JSON{Raw: []byte(modelOnlyCarrier)}, + }, + } +} + +func aisimulateImageOf(t *testing.T, raw []byte) string { + t.Helper() + var spec map[string]any + if err := json.Unmarshal(raw, &spec); err != nil { + t.Fatalf("decode carrier: %v", err) + } + aisimulate, _ := spec["aisimulate"].(map[string]any) + image, _ := aisimulate["image"].(string) + return image +} + +func TestDGDRDefaulter_RoutesByRequestedVersion(t *testing.T) { + d := NewDGDRDefaulter("1.1.0", "").WithV1beta2(DGDRV1beta2Options{AISimulateImage: "aisim:1"}) + + t.Run("a native v1beta2 request is defaulted from its carrier", func(t *testing.T) { + dgdr := carrierRequest() + if err := d.Default(ctxForRequest(admissionv1.Create, "v1beta2"), dgdr); err != nil { + t.Fatalf("Default() error = %v", err) + } + if got := aisimulateImageOf(t, dgdr.Spec.V1Beta2.Raw); got != "aisim:1" { + t.Fatalf("aisimulate.image = %q, want aisim:1", got) + } + }) + + t.Run("RequestKind decides, not the converted Kind", func(t *testing.T) { + dgdr := carrierRequest() + if err := d.Default(ctxForRequest(admissionv1.Create, "v1beta2"), dgdr); err != nil { + t.Fatalf("Default() error = %v", err) + } + if dgdr.Spec.Image != "" { + t.Fatalf("v1beta1 image defaulting touched a v1beta2 request: %q", dgdr.Spec.Image) + } + }) + + for _, version := range []string{"", "v1beta1", "v1alpha1"} { + t.Run("a foreign carrier is never mutated, requested as "+version, func(t *testing.T) { + for _, op := range []admissionv1.Operation{admissionv1.Create, admissionv1.Update} { + dgdr := carrierRequest() + before := append([]byte(nil), dgdr.Spec.V1Beta2.Raw...) + if err := d.Default(ctxForRequest(op, version), dgdr); err != nil { + t.Fatalf("Default() error = %v", err) + } + if !bytes.Equal(dgdr.Spec.V1Beta2.Raw, before) { + t.Fatalf("%s: carrier changed from %s to %s", op, before, dgdr.Spec.V1Beta2.Raw) + } + if dgdr.Spec.Image != "" { + t.Fatalf("%s: native v1beta1 defaults reached a foreign carrier", op) + } + } + }) + } + + t.Run("a v1beta1 payload written through v1beta2 is left for validation", func(t *testing.T) { + dgdr := &nvidiacomv1beta1.DynamoGraphDeploymentRequest{Spec: nvidiacomv1beta1.DynamoGraphDeploymentRequestSpec{Model: "m"}} + if err := d.Default(ctxForRequest(admissionv1.Create, "v1beta2"), dgdr); err != nil { + t.Fatalf("Default() error = %v", err) + } + if dgdr.Spec.Image != "" { + t.Fatalf("a foreign v1beta1 payload was defaulted: image %q", dgdr.Spec.Image) + } + }) + + t.Run("a native v1beta1 request keeps its defaults", func(t *testing.T) { + dgdr := &nvidiacomv1beta1.DynamoGraphDeploymentRequest{Spec: nvidiacomv1beta1.DynamoGraphDeploymentRequestSpec{Model: "m"}} + if err := d.Default(ctxForRequest(admissionv1.Create, ""), dgdr); err != nil { + t.Fatalf("Default() error = %v", err) + } + if dgdr.Spec.Image == "" { + t.Fatal("native v1beta1 image defaulting stopped working") + } + }) +} diff --git a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta1_options.go b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta1_options.go new file mode 100644 index 000000000000..f5140ee376a0 --- /dev/null +++ b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta1_options.go @@ -0,0 +1,13 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package defaulting + +// DGDRV1beta1Options configures the defaults of v1beta1 Requests. +type DGDRV1beta1Options struct { + // DefaultImage is the profiler image put into spec.image when a Request is created + // without one; empty derives dynamo-planner:. + DefaultImage string +} diff --git a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2.go b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2.go new file mode 100644 index 000000000000..e8a6eebbc89d --- /dev/null +++ b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2.go @@ -0,0 +1,196 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package defaulting + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + + nvidiacomv1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" + nvidiacomv1beta2 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta2" + admissionv1 "k8s.io/api/admission/v1" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" +) + +// defaultMetricV1beta2 is the scalar target an Optimize objective receives when it names none. +const defaultMetricV1beta2 = nvidiacomv1beta2.ObjectiveMetric("Throughput") + +// Nested defaults of the parents admission creates. They must equal the +kubebuilder:default +// markers in api/v1beta2: the API server defaults a mutated object in the webhook's version +// (v1beta1, whose v1beta2 carrier is opaque), so the v1beta2 schema never fills them in. +const ( + defaultStaticISLV1beta2 int32 = 1024 + defaultStaticOSLV1beta2 int32 = 128 + defaultStaticConcurrencyV1beta2 int32 = 10 + defaultMaxCandidatesV1beta2 int32 = 1 +) + +func defaultPtrV1beta2[T any](v T) *T { return &v } + +// This file defaults the v1beta2 shape of a DynamoGraphDeploymentRequest. It is separate +// from the canonical defaulting because the v1beta1 hub carries a v1beta2 Request as an +// opaque spec.v1beta2 envelope: conversion cannot apply field-level defaults to it, so the +// handler decodes the envelope, runs this traversal and encodes the result. Names carry the +// API version because the canonical names belong to v1beta1. +// +// The CRD schema owns the unconditional defaults: backends, objective.mode, +// recommendation.maxCandidates and workload.static as a whole (workload.static: {} opts +// out). objective.sla has no default: an omitted SLA stays absent. Admission owns what depends on +// other fields or on the environment: the scalar metric, the cluster hardware and the +// AISimulate image. spec.aisimulate.parameters never changes a default. It overrides the +// defaulted typed values later, when the input is lowered. + +// defaultDynamoGraphDeploymentRequestV1beta2 defaults the v1beta2 spec that dgdr carries in +// spec.v1beta2. The environment is read once, before any mutation, and only on CREATE: +// discovered hardware is persisted with the Request, so a later write never replaces what +// the Request was created with. The envelope is rewritten +// only when defaulting changed the spec, so a defaulted object stays byte-identical across +// repeated admissions. +func defaultDynamoGraphDeploymentRequestV1beta2( + ctx context.Context, + dgdr *nvidiacomv1beta1.DynamoGraphDeploymentRequest, + operation admissionv1.Operation, + options DGDRV1beta2Options, +) error { + var spec nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec + if err := json.Unmarshal(dgdr.Spec.V1Beta2.Raw, &spec); err != nil { + return fmt.Errorf("decode spec.v1beta2: %w", err) + } + before, err := json.Marshal(&spec) + if err != nil { + return fmt.Errorf("encode spec.v1beta2: %w", err) + } + + defaultDynamoGraphDeploymentRequestSpecV1beta2(&spec, options.resolveDefaults(ctx, operation, &spec)) + + after, err := json.Marshal(&spec) + if err != nil { + return fmt.Errorf("encode spec.v1beta2: %w", err) + } + if !bytes.Equal(before, after) { + dgdr.Spec.V1Beta2 = &apiextensionsv1.JSON{Raw: after} + } + return nil +} + +// dynamoGraphDeploymentRequestV1beta2DefaultOptions carries the request-scoped values the +// traversal needs. An empty field means the environment had nothing to offer. +type dynamoGraphDeploymentRequestV1beta2DefaultOptions struct { + // hardware is the GPU identity and budget discovered from the cluster. + hardware hardwareDefaultsV1beta2 + // aisimulateImage is the operator's configured AISimulate image. + aisimulateImage string +} + +// hardwareDefaultsV1beta2 is the hardware the cluster can offer for the values a Request omits. +type hardwareDefaultsV1beta2 struct { + sku string + budget int32 +} + +func (h hardwareDefaultsV1beta2) empty() bool { return h.sku == "" && h.budget == 0 } + +// defaultDynamoGraphDeploymentRequestSpecV1beta2 defaults spec in place. It applies on +// CREATE and UPDATE and is idempotent; opts carries no hardware on UPDATE. spec must not be nil. +func defaultDynamoGraphDeploymentRequestSpecV1beta2( + spec *nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec, + opts dynamoGraphDeploymentRequestV1beta2DefaultOptions, +) { + // Structural defaults live here, not in the CRD schema: a schema default would also be + // applied to an object carrying spec.v1beta1 and put native fields next to the carrier, + // which conversion rejects. This function only runs for native requests. Explicit values, + // including empty objects, are kept. A parent the client sent already received its nested + // schema defaults at decode; a parent created here gets them explicitly. + if spec.Backends == nil { + spec.Backends = []nvidiacomv1beta2.Backend{"vllm", "sglang", "trtllm"} + } + if spec.Workload == nil { + spec.Workload = &nvidiacomv1beta2.WorkloadSpec{Static: &nvidiacomv1beta2.StaticWorkloadSpec{ + ISL: defaultPtrV1beta2(defaultStaticISLV1beta2), + OSL: defaultPtrV1beta2(defaultStaticOSLV1beta2), + Concurrency: defaultPtrV1beta2(defaultStaticConcurrencyV1beta2), + }} + } + if spec.Objective == nil { + spec.Objective = &nvidiacomv1beta2.ObjectiveSpec{Mode: nvidiacomv1beta2.ObjectiveModeOptimize} + } + if spec.Recommendation == nil { + spec.Recommendation = &nvidiacomv1beta2.RecommendationSpec{MaxCandidates: defaultMaxCandidatesV1beta2} + } + + if spec.Hardware == nil && !opts.hardware.empty() { + spec.Hardware = &nvidiacomv1beta2.HardwareSpec{} + } + if spec.Hardware != nil { + defaultHardwareSpecV1beta2(spec.Hardware, opts.hardware) + } + + if spec.Objective != nil { + defaultObjectiveSpecV1beta2(spec.Objective) + } + + // The operator image is persisted with the Request, so spec.aisimulate is created to + // carry it. + if spec.AISimulate == nil && opts.aisimulateImage != "" { + spec.AISimulate = &nvidiacomv1beta2.AISimulateSpec{} + } + if spec.AISimulate != nil { + defaultAISimulateSpecV1beta2(spec.AISimulate, opts.aisimulateImage) + } +} + +func defaultHardwareSpecV1beta2(hardware *nvidiacomv1beta2.HardwareSpec, defaults hardwareDefaultsV1beta2) { + if hardware.GPU == nil && !defaults.empty() { + hardware.GPU = &nvidiacomv1beta2.GPUHardwareSpec{} + } + if hardware.GPU != nil { + defaultGPUHardwareSpecV1beta2(hardware.GPU, defaults) + } +} + +// defaultGPUHardwareSpecV1beta2 fills each omitted value on its own, so a caller can pin the +// SKU and let admission size the budget. +func defaultGPUHardwareSpecV1beta2(gpu *nvidiacomv1beta2.GPUHardwareSpec, defaults hardwareDefaultsV1beta2) { + if gpu.SKU == nil && defaults.sku != "" { + sku := defaults.sku + gpu.SKU = &sku + } + if gpu.Budget == nil && defaults.budget > 0 { + budget := defaults.budget + gpu.Budget = &budget + } +} + +// defaultObjectiveSpecV1beta2 gives an Optimize objective without a metric the default +// scalar target. It reads the typed objective alone: a goal.target in +// spec.aisimulate.parameters overrides the completed typed value when the input is lowered, +// so it never suppresses this default. +func defaultObjectiveSpecV1beta2(objective *nvidiacomv1beta2.ObjectiveSpec) { + if (objective.Mode == "" || objective.Mode == nvidiacomv1beta2.ObjectiveModeOptimize) && objective.Metric == nil { + metric := defaultMetricV1beta2 + objective.Metric = &metric + } +} + +func defaultAISimulateSpecV1beta2(aisimulate *nvidiacomv1beta2.AISimulateSpec, image string) { + if aisimulate.Image == "" && image != "" { + aisimulate.Image = image + } +} diff --git a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_crd_test.go b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_crd_test.go new file mode 100644 index 000000000000..804761355ae9 --- /dev/null +++ b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_crd_test.go @@ -0,0 +1,92 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package defaulting + +import ( + "encoding/json" + "os" + "path/filepath" + "reflect" + "runtime" + "testing" + + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + "sigs.k8s.io/yaml" +) + +// v1beta2SpecSchema returns the spec schema of the generated v1beta2 Request CRD. +func v1beta2SpecSchema(t *testing.T) apiextensionsv1.JSONSchemaProps { + t.Helper() + _, thisFile, _, _ := runtime.Caller(0) + path := filepath.Join(filepath.Dir(thisFile), "../../../config/crd/bases", "nvidia.com_dynamographdeploymentrequests.yaml") + raw, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read the generated CRD: %v", err) + } + var crd apiextensionsv1.CustomResourceDefinition + if err := yaml.Unmarshal(raw, &crd); err != nil { + t.Fatalf("decode the generated CRD: %v", err) + } + for _, version := range crd.Spec.Versions { + if version.Name == "v1beta2" { + return version.Schema.OpenAPIV3Schema.Properties["spec"] + } + } + t.Fatal("the generated CRD has no v1beta2 version") + return apiextensionsv1.JSONSchemaProps{} +} + +func schemaDefault(t *testing.T, prop apiextensionsv1.JSONSchemaProps) any { + t.Helper() + if prop.Default == nil { + return nil + } + var value any + if err := json.Unmarshal(prop.Default.Raw, &value); err != nil { + t.Fatalf("decode a schema default: %v", err) + } + return value +} + +// A schema default on a top-level native field is also applied to an object that carries +// spec.v1beta1, which puts native fields next to the carrier and makes conversion reject it. +func TestV1beta2CRDHasNoTopLevelStructuralDefaults(t *testing.T) { + spec := v1beta2SpecSchema(t) + for _, field := range []string{"backends", "workload", "objective", "recommendation"} { + prop, ok := spec.Properties[field] + if !ok { + t.Fatalf("spec.%s is missing from the v1beta2 schema", field) + } + if prop.Default != nil { + t.Errorf("spec.%s has a schema default %s; top-level defaults belong in admission", field, string(prop.Default.Raw)) + } + } +} + +// Admission creates workload, objective and recommendation itself, and the API server then +// defaults the mutated object in the webhook's version, so the nested schema defaults would +// not apply to them. The webhook sets those values explicitly and they must equal the schema. +func TestV1beta2NestedDefaultsMatchTheCRD(t *testing.T) { + spec := v1beta2SpecSchema(t) + + wantStatic := map[string]any{ + "isl": float64(defaultStaticISLV1beta2), + "osl": float64(defaultStaticOSLV1beta2), + "concurrency": float64(defaultStaticConcurrencyV1beta2), + } + got := schemaDefault(t, spec.Properties["workload"].Properties["static"]) + if !reflect.DeepEqual(got, wantStatic) { + t.Errorf("workload.static default = %v, webhook constants = %v", got, wantStatic) + } + + if got := schemaDefault(t, spec.Properties["objective"].Properties["mode"]); got != "Optimize" { + t.Errorf("objective.mode default = %v, webhook sets Optimize", got) + } + + if got := schemaDefault(t, spec.Properties["recommendation"].Properties["maxCandidates"]); got != float64(defaultMaxCandidatesV1beta2) { + t.Errorf("recommendation.maxCandidates default = %v, webhook constant = %d", got, defaultMaxCandidatesV1beta2) + } +} diff --git a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_helpers.go b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_helpers.go new file mode 100644 index 000000000000..31a9996534ea --- /dev/null +++ b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_helpers.go @@ -0,0 +1,159 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package defaulting + +import ( + "context" + "time" + + nvidiacomv1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" + nvidiacomv1beta2 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta2" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/gpu" + admissionv1 "k8s.io/api/admission/v1" + "sigs.k8s.io/controller-runtime/pkg/log" +) + +const ( + // defaultMaxHardwareBudget caps a discovered GPU budget. It matches the limit DGDR + // v1beta1 applies to auto-discovered hardware, which keeps a search from using every GPU + // in a large cluster. Set a budget explicitly to exceed it. + defaultMaxHardwareBudget int32 = 32 + + // defaultHardwareDiscoveryTimeout bounds GPU discovery inside the admission call. Keep it + // below the webhook's timeoutSeconds, or the API server gives up first and the request + // fails under the webhook's failurePolicy instead of reaching validation. + defaultHardwareDiscoveryTimeout = 5 * time.Second +) + +// DiscoveredHardware is what the cluster has, as far as it can be determined. +type DiscoveredHardware struct { + // SKU is the AISimulate hardware catalog name, or empty when the GPU model is not one + // the catalog knows. + SKU string + // GPUs is the total number of GPUs of that SKU in the cluster. + GPUs int +} + +// HardwareDiscoverer finds the cluster's GPUs. filterSKU, when set, restricts the result to +// GPUs of that SKU. +type HardwareDiscoverer interface { + Discover(ctx context.Context, filterSKU string) (DiscoveredHardware, error) +} + +// DGDRV1beta2Options configures the defaults of v1beta2 Requests. +type DGDRV1beta2Options struct { + // AISimulateImage is the operator's AISimulate runner image, persisted into + // spec.aisimulate.image when the caller omits it. Empty persists nothing, and + // validation then requires the image to be spelled out. + AISimulateImage string + // Hardware discovers the cluster's GPUs. Nil disables hardware defaulting, and + // validation then requires the hardware to be spelled out. + Hardware HardwareDiscoverer + // HardwareDiscoveryTimeout defaults to 5 seconds. + HardwareDiscoveryTimeout time.Duration + // MaxHardwareBudget caps a discovered GPU budget; zero means 32. + MaxHardwareBudget int32 +} + +// resolveDefaults reads the environment for the values a v1beta2 spec omits. It does not +// mutate spec, and it never fails: a value the environment cannot supply stays unset and +// validating admission reports the exact field. +func (o DGDRV1beta2Options) resolveDefaults( + ctx context.Context, + operation admissionv1.Operation, + spec *nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec, +) dynamoGraphDeploymentRequestV1beta2DefaultOptions { + return dynamoGraphDeploymentRequestV1beta2DefaultOptions{ + hardware: o.resolveHardware(ctx, operation, spec), + aisimulateImage: o.AISimulateImage, + } +} + +// resolveHardware returns the SKU and budget the typed spec omits on CREATE. It asks the cluster only +// when something is missing, restricted to the typed SKU when there is one. +func (o DGDRV1beta2Options) resolveHardware( + ctx context.Context, + operation admissionv1.Operation, + spec *nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec, +) hardwareDefaultsV1beta2 { + // Discovery is a CREATE-time decision. A Request that omitted its hardware and was + // created while discovery had nothing to offer must not pick hardware up on a later + // metadata-only update: that would change the spec, and its generation, behind the + // writer's back. Any other operation, known or not, leaves the hardware alone. + if operation != admissionv1.Create || o.Hardware == nil { + return hardwareDefaultsV1beta2{} + } + var sku string + var hasBudget bool + if spec.Hardware != nil && spec.Hardware.GPU != nil { + if spec.Hardware.GPU.SKU != nil { + sku = *spec.Hardware.GPU.SKU + } + hasBudget = spec.Hardware.GPU.Budget != nil + } + if sku != "" && hasBudget { + return hardwareDefaultsV1beta2{} + } + + timeout := o.HardwareDiscoveryTimeout + if timeout <= 0 { + timeout = defaultHardwareDiscoveryTimeout + } + ctx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + + found, err := o.Hardware.Discover(ctx, sku) + if err != nil { + log.FromContext(ctx).Info("hardware discovery failed; leaving hardware unresolved", "error", err.Error()) + return hardwareDefaultsV1beta2{} + } + + var defaults hardwareDefaultsV1beta2 + if sku == "" { + defaults.sku = found.SKU + } + if !hasBudget && found.GPUs > 0 { + maxBudget := o.MaxHardwareBudget + if maxBudget <= 0 { + maxBudget = defaultMaxHardwareBudget + } + defaults.budget = int32(min(found.GPUs, int(maxBudget))) + } + return defaults +} + +// gpuHardwareDiscoverer backs HardwareDiscoverer with the operator's GPU discovery. +type gpuHardwareDiscoverer struct { + discoverer *gpu.Discoverer +} + +// NewGPUHardwareDiscoverer adapts the operator's GPU discovery for hardware defaulting. +func NewGPUHardwareDiscoverer(discoverer *gpu.Discoverer) HardwareDiscoverer { + return gpuHardwareDiscoverer{discoverer: discoverer} +} + +func (g gpuHardwareDiscoverer) Discover(ctx context.Context, filterSKU string) (DiscoveredHardware, error) { + info, err := g.discoverer.Discover(ctx, nvidiacomv1beta1.GPUSKUType(filterSKU)) + if err != nil { + return DiscoveredHardware{}, err + } + return DiscoveredHardware{ + SKU: string(gpu.KnownSKU(info)), + GPUs: info.GPUsPerNode * info.NodesWithGPUs, + }, nil +} diff --git a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_test.go b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_test.go new file mode 100644 index 000000000000..decb7fc49b23 --- /dev/null +++ b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_test.go @@ -0,0 +1,488 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package defaulting + +import ( + "context" + "encoding/json" + "errors" + "testing" + "time" + + nvidiacomv1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" + nvidiacomv1beta2 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta2" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/controller/aisimulate" + admissionv1 "k8s.io/api/admission/v1" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + "k8s.io/apimachinery/pkg/runtime" +) + +func v1beta2Ptr[T any](v T) *T { return &v } + +// v1beta2SchemaDefaulted is a model-only Request as the API server hands it to admission: the +// CRD has applied its defaults, and admission has not run. +func v1beta2SchemaDefaulted() *nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec { + return &nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec{ + Model: &nvidiacomv1beta2.ModelSpec{Name: "meta-llama/Llama-3.1-8B-Instruct"}, + Backends: []nvidiacomv1beta2.Backend{nvidiacomv1beta2.BackendVLLM, nvidiacomv1beta2.BackendSGLang, nvidiacomv1beta2.BackendTRTLLM}, + Workload: &nvidiacomv1beta2.WorkloadSpec{Static: &nvidiacomv1beta2.StaticWorkloadSpec{ + ISL: v1beta2Ptr(int32(1024)), OSL: v1beta2Ptr(int32(128)), Concurrency: v1beta2Ptr(int32(10)), + }}, + Objective: &nvidiacomv1beta2.ObjectiveSpec{ + Mode: nvidiacomv1beta2.ObjectiveModeOptimize, + }, + Recommendation: &nvidiacomv1beta2.RecommendationSpec{MaxCandidates: 1}, + } +} + +func v1beta2JSON(t *testing.T, spec *nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec) string { + t.Helper() + raw, err := json.Marshal(spec) + if err != nil { + t.Fatal(err) + } + return string(raw) +} + +func TestDefaultSpecV1beta2ModelOnlyRequest(t *testing.T) { + spec := v1beta2SchemaDefaulted() + defaultDynamoGraphDeploymentRequestSpecV1beta2(spec, dynamoGraphDeploymentRequestV1beta2DefaultOptions{ + hardware: hardwareDefaultsV1beta2{sku: "h200_sxm", budget: 8}, + aisimulateImage: "registry.example.com/aisimulate:1.2.0", + }) + if got := ptrDeref(spec.Objective.Metric); got != "Throughput" { + t.Fatalf("metric = %q", got) + } + gpu := spec.Hardware.GPU + if *gpu.SKU != "h200_sxm" || *gpu.Budget != 8 { + t.Fatalf("hardware = %s/%d", *gpu.SKU, *gpu.Budget) + } + if spec.AISimulate.Image != "registry.example.com/aisimulate:1.2.0" { + t.Fatalf("image = %q", spec.AISimulate.Image) + } +} + +func ptrDeref(metric *nvidiacomv1beta2.ObjectiveMetric) nvidiacomv1beta2.ObjectiveMetric { + if metric == nil { + return "" + } + return *metric +} + +func TestDefaultSpecV1beta2IsIdempotent(t *testing.T) { + opts := dynamoGraphDeploymentRequestV1beta2DefaultOptions{ + hardware: hardwareDefaultsV1beta2{sku: "h200_sxm", budget: 8}, + aisimulateImage: "registry.example.com/aisimulate:1.2.0", + } + spec := v1beta2SchemaDefaulted() + defaultDynamoGraphDeploymentRequestSpecV1beta2(spec, opts) + first := v1beta2JSON(t, spec) + defaultDynamoGraphDeploymentRequestSpecV1beta2(spec, opts) + if second := v1beta2JSON(t, spec); second != first { + t.Fatalf("a second pass changed the spec:\n%s\n%s", first, second) + } +} + +func TestDefaultSpecV1beta2PreservesExplicitValues(t *testing.T) { + opts := dynamoGraphDeploymentRequestV1beta2DefaultOptions{ + hardware: hardwareDefaultsV1beta2{sku: "h200_sxm", budget: 8}, + aisimulateImage: "registry.example.com/aisimulate:1.2.0", + } + t.Run("hardware", func(t *testing.T) { + spec := v1beta2SchemaDefaulted() + spec.Hardware = &nvidiacomv1beta2.HardwareSpec{GPU: &nvidiacomv1beta2.GPUHardwareSpec{SKU: v1beta2Ptr("b200_sxm"), Budget: v1beta2Ptr(int32(2))}} + defaultDynamoGraphDeploymentRequestSpecV1beta2(spec, opts) + if *spec.Hardware.GPU.SKU != "b200_sxm" || *spec.Hardware.GPU.Budget != 2 { + t.Fatalf("hardware overwritten: %s/%d", *spec.Hardware.GPU.SKU, *spec.Hardware.GPU.Budget) + } + }) + t.Run("a pinned SKU lets admission size the budget", func(t *testing.T) { + spec := v1beta2SchemaDefaulted() + spec.Hardware = &nvidiacomv1beta2.HardwareSpec{GPU: &nvidiacomv1beta2.GPUHardwareSpec{SKU: v1beta2Ptr("b200_sxm")}} + defaultDynamoGraphDeploymentRequestSpecV1beta2(spec, opts) + if *spec.Hardware.GPU.SKU != "b200_sxm" || *spec.Hardware.GPU.Budget != 8 { + t.Fatalf("got %s/%d", *spec.Hardware.GPU.SKU, *spec.Hardware.GPU.Budget) + } + }) + t.Run("an empty hardware object is filled", func(t *testing.T) { + spec := v1beta2SchemaDefaulted() + spec.Hardware = &nvidiacomv1beta2.HardwareSpec{} + defaultDynamoGraphDeploymentRequestSpecV1beta2(spec, opts) + if spec.Hardware.GPU == nil || *spec.Hardware.GPU.SKU != "h200_sxm" { + t.Fatal("hardware not filled") + } + }) + t.Run("metric", func(t *testing.T) { + spec := v1beta2SchemaDefaulted() + spec.Objective.Metric = v1beta2Ptr(nvidiacomv1beta2.ObjectiveMetric("TTFT")) + defaultDynamoGraphDeploymentRequestSpecV1beta2(spec, opts) + if *spec.Objective.Metric != "TTFT" { + t.Fatalf("metric overwritten: %s", *spec.Objective.Metric) + } + }) + t.Run("image", func(t *testing.T) { + spec := v1beta2SchemaDefaulted() + spec.AISimulate = &nvidiacomv1beta2.AISimulateSpec{Image: "registry.example.com/aisimulate:custom", VersionOverride: "1.4.0"} + defaultDynamoGraphDeploymentRequestSpecV1beta2(spec, opts) + if spec.AISimulate.Image != "registry.example.com/aisimulate:custom" { + t.Fatalf("image overwritten: %s", spec.AISimulate.Image) + } + }) + t.Run("parameters are untouched", func(t *testing.T) { + spec := v1beta2SchemaDefaulted() + spec.AISimulate = &nvidiacomv1beta2.AISimulateSpec{Parameters: &runtime.RawExtension{Raw: []byte(`{"goal":{"target":"ttft"}}`)}} + defaultDynamoGraphDeploymentRequestSpecV1beta2(spec, opts) + if string(spec.AISimulate.Parameters.Raw) != `{"goal":{"target":"ttft"}}` { + t.Fatal("parameters changed") + } + if spec.AISimulate.Image != "registry.example.com/aisimulate:1.2.0" { + t.Fatal("image not persisted next to parameters") + } + }) +} + +func TestDefaultSpecV1beta2NeverTouchesTheSchemaOwnedValues(t *testing.T) { + opts := dynamoGraphDeploymentRequestV1beta2DefaultOptions{ + hardware: hardwareDefaultsV1beta2{sku: "h200_sxm", budget: 8}, + aisimulateImage: "registry.example.com/aisimulate:1.2.0", + } + t.Run("explicit empty workload", func(t *testing.T) { + spec := v1beta2SchemaDefaulted() + spec.Workload = &nvidiacomv1beta2.WorkloadSpec{} + defaultDynamoGraphDeploymentRequestSpecV1beta2(spec, opts) + if spec.Workload.Static != nil { + t.Fatal("an explicit workload: {} was filled") + } + }) + t.Run("omitted workload", func(t *testing.T) { + spec := v1beta2SchemaDefaulted() + spec.Workload = nil + defaultDynamoGraphDeploymentRequestSpecV1beta2(spec, opts) + if !hasModelOnlyStatic(spec.Workload) { + t.Fatal("an omitted workload must become workload: {}, with the static model-only default") + } + }) + t.Run("explicit empty SLA", func(t *testing.T) { + spec := v1beta2SchemaDefaulted() + spec.Objective.SLA = &nvidiacomv1beta2.ObjectiveSLASpec{} + defaultDynamoGraphDeploymentRequestSpecV1beta2(spec, opts) + if spec.Objective.SLA.TTFTMs != nil || spec.Objective.SLA.ITLMs != nil { + t.Fatal("an explicit sla: {} was filled") + } + }) + t.Run("an explicit zero-valued backend list element is kept", func(t *testing.T) { + spec := v1beta2SchemaDefaulted() + spec.Backends = []nvidiacomv1beta2.Backend{nvidiacomv1beta2.BackendVLLM} + defaultDynamoGraphDeploymentRequestSpecV1beta2(spec, opts) + if len(spec.Backends) != 1 { + t.Fatal("backends changed") + } + }) +} + +func TestDefaultObjectiveSpecV1beta2(t *testing.T) { + cases := []struct { + name string + objective nvidiacomv1beta2.ObjectiveSpec + want nvidiacomv1beta2.ObjectiveMetric + }{ + {"optimize without a metric", nvidiacomv1beta2.ObjectiveSpec{Mode: nvidiacomv1beta2.ObjectiveModeOptimize}, "Throughput"}, + {"an explicit metric is kept", nvidiacomv1beta2.ObjectiveSpec{Mode: nvidiacomv1beta2.ObjectiveModeOptimize, Metric: v1beta2Ptr(nvidiacomv1beta2.ObjectiveMetric("TTFT"))}, "TTFT"}, + {"pareto has no scalar metric", nvidiacomv1beta2.ObjectiveSpec{Mode: nvidiacomv1beta2.ObjectiveModePareto, Metrics: []nvidiacomv1beta2.ObjectiveMetric{"TTFT", "Throughput"}}, ""}, + {"an absent mode is Optimize, which the schema fills after admission", nvidiacomv1beta2.ObjectiveSpec{}, "Throughput"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + defaultObjectiveSpecV1beta2(&tc.objective) + if got := ptrDeref(tc.objective.Metric); got != tc.want { + t.Fatalf("metric = %q, want %q", got, tc.want) + } + }) + } +} + +func TestDefaultSpecV1beta2DoesNotCreateWhatTheEnvironmentCannotFill(t *testing.T) { + spec := v1beta2SchemaDefaulted() + defaultDynamoGraphDeploymentRequestSpecV1beta2(spec, dynamoGraphDeploymentRequestV1beta2DefaultOptions{}) + if spec.Hardware != nil || spec.AISimulate != nil { + t.Fatalf("created sections with nothing to put in them: %s", v1beta2JSON(t, spec)) + } + spec.Objective = nil + defaultDynamoGraphDeploymentRequestSpecV1beta2(spec, dynamoGraphDeploymentRequestV1beta2DefaultOptions{}) + if spec.Objective == nil || ptrDeref(spec.Objective.Metric) != "Throughput" { + t.Fatal("an omitted objective must become objective: {} with the Optimize metric default") + } +} + +func TestNativeTargetOverridesTheDefaultedMetric(t *testing.T) { + spec := v1beta2SchemaDefaulted() + spec.AISimulate = &nvidiacomv1beta2.AISimulateSpec{Parameters: &runtime.RawExtension{Raw: []byte(`{"goal":{"target":"ttft"}}`)}} + defaultDynamoGraphDeploymentRequestSpecV1beta2(spec, dynamoGraphDeploymentRequestV1beta2DefaultOptions{}) + if *spec.Objective.Metric != "Throughput" { + t.Fatalf("the typed metric must be defaulted regardless of the parameters, got %q", *spec.Objective.Metric) + } + input, err := aisimulate.Lower(spec) + if err != nil { + t.Fatal(err) + } + if target, _ := input["goal"].(map[string]any)["target"]; target != "ttft" { + t.Fatalf("the native target must win when lowered, got %v", target) + } +} + +type fakeDiscoverer struct { + found DiscoveredHardware + err error + calls int + filter string + timeout time.Duration +} + +func (f *fakeDiscoverer) Discover(ctx context.Context, filterSKU string) (DiscoveredHardware, error) { + f.calls++ + f.filter = filterSKU + if deadline, ok := ctx.Deadline(); ok { + f.timeout = time.Until(deadline) + } + return f.found, f.err +} + +func TestResolveHardwareV1beta2(t *testing.T) { + spec := func(sku *string, budget *int32) *nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec { + s := v1beta2SchemaDefaulted() + if sku != nil || budget != nil { + s.Hardware = &nvidiacomv1beta2.HardwareSpec{GPU: &nvidiacomv1beta2.GPUHardwareSpec{SKU: sku, Budget: budget}} + } + return s + } + + t.Run("no discoverer", func(t *testing.T) { + if got := (DGDRV1beta2Options{}).resolveHardware(context.Background(), admissionv1.Create, spec(nil, nil)); !got.empty() { + t.Fatalf("got %+v", got) + } + }) + t.Run("complete hardware never asks the cluster", func(t *testing.T) { + d := &fakeDiscoverer{found: DiscoveredHardware{SKU: "h100_sxm", GPUs: 4}} + got := DGDRV1beta2Options{Hardware: d}.resolveHardware(context.Background(), admissionv1.Create, spec(v1beta2Ptr("b200_sxm"), v1beta2Ptr(int32(2)))) + if d.calls != 0 || !got.empty() { + t.Fatalf("calls=%d got=%+v", d.calls, got) + } + }) + t.Run("everything missing", func(t *testing.T) { + d := &fakeDiscoverer{found: DiscoveredHardware{SKU: "h200_sxm", GPUs: 16}} + got := DGDRV1beta2Options{Hardware: d}.resolveHardware(context.Background(), admissionv1.Create, spec(nil, nil)) + if got.sku != "h200_sxm" || got.budget != 16 || d.filter != "" { + t.Fatalf("got %+v, filter %q", got, d.filter) + } + if d.timeout <= 0 || d.timeout > defaultHardwareDiscoveryTimeout { + t.Fatalf("discovery timeout = %v", d.timeout) + } + }) + t.Run("a pinned SKU filters discovery and keeps its value", func(t *testing.T) { + d := &fakeDiscoverer{found: DiscoveredHardware{SKU: "b200_sxm", GPUs: 4}} + got := DGDRV1beta2Options{Hardware: d}.resolveHardware(context.Background(), admissionv1.Create, spec(v1beta2Ptr("b200_sxm"), nil)) + if d.filter != "b200_sxm" || got.sku != "" || got.budget != 4 { + t.Fatalf("got %+v, filter %q", got, d.filter) + } + }) + t.Run("a pinned budget keeps its value", func(t *testing.T) { + d := &fakeDiscoverer{found: DiscoveredHardware{SKU: "h200_sxm", GPUs: 64}} + got := DGDRV1beta2Options{Hardware: d}.resolveHardware(context.Background(), admissionv1.Create, spec(nil, v1beta2Ptr(int32(3)))) + if got.sku != "h200_sxm" || got.budget != 0 { + t.Fatalf("got %+v", got) + } + }) + t.Run("the budget is capped", func(t *testing.T) { + d := &fakeDiscoverer{found: DiscoveredHardware{SKU: "h200_sxm", GPUs: 1024}} + if got := (DGDRV1beta2Options{Hardware: d}).resolveHardware(context.Background(), admissionv1.Create, spec(nil, nil)); got.budget != 32 { + t.Fatalf("default cap: %+v", got) + } + if got := (DGDRV1beta2Options{Hardware: d, MaxHardwareBudget: 8}).resolveHardware(context.Background(), admissionv1.Create, spec(nil, nil)); got.budget != 8 { + t.Fatalf("explicit cap: %+v", got) + } + }) + t.Run("an unknown GPU model yields no SKU", func(t *testing.T) { + d := &fakeDiscoverer{found: DiscoveredHardware{GPUs: 8}} + if got := (DGDRV1beta2Options{Hardware: d}).resolveHardware(context.Background(), admissionv1.Create, spec(nil, nil)); got.sku != "" || got.budget != 8 { + t.Fatalf("got %+v", got) + } + }) + t.Run("a discovery failure leaves everything unset", func(t *testing.T) { + d := &fakeDiscoverer{err: errors.New("no DCGM")} + if got := (DGDRV1beta2Options{Hardware: d}).resolveHardware(context.Background(), admissionv1.Create, spec(nil, nil)); !got.empty() { + t.Fatalf("got %+v", got) + } + }) + t.Run("the timeout is configurable", func(t *testing.T) { + d := &fakeDiscoverer{found: DiscoveredHardware{GPUs: 1}} + DGDRV1beta2Options{Hardware: d, HardwareDiscoveryTimeout: time.Hour}.resolveHardware(context.Background(), admissionv1.Create, spec(nil, nil)) + if d.timeout < time.Minute { + t.Fatalf("timeout = %v", d.timeout) + } + }) +} + +func TestResolveDefaultsV1beta2(t *testing.T) { + d := &fakeDiscoverer{found: DiscoveredHardware{SKU: "h200_sxm", GPUs: 8}} + got := DGDRV1beta2Options{Hardware: d, AISimulateImage: "registry.example.com/aisimulate:1.2.0"}. + resolveDefaults(context.Background(), admissionv1.Create, v1beta2SchemaDefaulted()) + if got.aisimulateImage != "registry.example.com/aisimulate:1.2.0" || got.hardware.sku != "h200_sxm" || got.hardware.budget != 8 { + t.Fatalf("got %+v", got) + } +} + +func v1beta2Envelope(t *testing.T, spec *nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec) *nvidiacomv1beta1.DynamoGraphDeploymentRequest { + t.Helper() + raw, err := json.Marshal(spec) + if err != nil { + t.Fatal(err) + } + dgdr := &nvidiacomv1beta1.DynamoGraphDeploymentRequest{} + dgdr.Spec.V1Beta2 = &apiextensionsv1.JSON{Raw: raw} + return dgdr +} + +func TestDefaultEnvelopeV1beta2(t *testing.T) { + options := DGDRV1beta2Options{ + AISimulateImage: "registry.example.com/aisimulate:1.2.0", + Hardware: &fakeDiscoverer{found: DiscoveredHardware{SKU: "h200_sxm", GPUs: 8}}, + } + dgdr := v1beta2Envelope(t, v1beta2SchemaDefaulted()) + + if err := defaultDynamoGraphDeploymentRequestV1beta2(context.Background(), dgdr, admissionv1.Create, options); err != nil { + t.Fatal(err) + } + var got nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec + if err := json.Unmarshal(dgdr.Spec.V1Beta2.Raw, &got); err != nil { + t.Fatal(err) + } + if got.Hardware == nil || got.Hardware.GPU == nil || *got.Hardware.GPU.SKU != "h200_sxm" || *got.Hardware.GPU.Budget != 8 { + t.Fatalf("hardware not defaulted: %+v", got.Hardware) + } + if got.AISimulate == nil || got.AISimulate.Image != "registry.example.com/aisimulate:1.2.0" { + t.Fatalf("image not defaulted: %+v", got.AISimulate) + } + if got.Objective == nil || got.Objective.Metric == nil { + t.Fatalf("objective metric not defaulted: %+v", got.Objective) + } + + // A second admission of the defaulted object changes nothing, byte for byte. + first := string(dgdr.Spec.V1Beta2.Raw) + if err := defaultDynamoGraphDeploymentRequestV1beta2(context.Background(), dgdr, admissionv1.Create, options); err != nil { + t.Fatal(err) + } + if string(dgdr.Spec.V1Beta2.Raw) != first { + t.Fatalf("not idempotent:\n%s\n%s", first, dgdr.Spec.V1Beta2.Raw) + } +} + +func TestDefaultEnvelopeV1beta2LeavesUnchangedEnvelopeUntouched(t *testing.T) { + // Nothing for admission to add: the objective already carries its metric and there is no + // environment to contribute. The envelope bytes stay exactly as sent, even though they are + // not in canonical form. + spec := v1beta2SchemaDefaulted() + spec.Objective.Metric = v1beta2Ptr(nvidiacomv1beta2.ObjectiveMetric("ThroughputPerGPU")) + raw, _ := json.Marshal(spec) + dgdr := v1beta2Envelope(t, spec) + dgdr.Spec.V1Beta2.Raw = append([]byte(" "), raw...) + original := string(dgdr.Spec.V1Beta2.Raw) + if err := defaultDynamoGraphDeploymentRequestV1beta2(context.Background(), dgdr, admissionv1.Create, DGDRV1beta2Options{}); err != nil { + t.Fatal(err) + } + if string(dgdr.Spec.V1Beta2.Raw) != original { + t.Fatalf("envelope was rewritten without a change: %q", dgdr.Spec.V1Beta2.Raw) + } +} + +func TestDefaultEnvelopeV1beta2RejectsMalformedSpec(t *testing.T) { + dgdr := &nvidiacomv1beta1.DynamoGraphDeploymentRequest{} + dgdr.Spec.V1Beta2 = &apiextensionsv1.JSON{Raw: []byte(`{"model": 3}`)} + if err := defaultDynamoGraphDeploymentRequestV1beta2(context.Background(), dgdr, admissionv1.Create, DGDRV1beta2Options{}); err == nil { + t.Fatal("want a decode error") + } +} + +func TestHardwareIsDiscoveredOnlyOnCreate(t *testing.T) { + for _, op := range []admissionv1.Operation{admissionv1.Update, admissionv1.Delete, admissionv1.Connect, ""} { + t.Run("operation "+string(op), func(t *testing.T) { + d := &fakeDiscoverer{found: DiscoveredHardware{SKU: "h200_sxm", GPUs: 8}} + options := DGDRV1beta2Options{Hardware: d} + if got := options.resolveHardware(context.Background(), op, v1beta2SchemaDefaulted()); !got.empty() { + t.Fatalf("hardware resolved on %q: %+v", op, got) + } + if d.calls != 0 { + t.Fatalf("the cluster was asked on %q", op) + } + }) + } +} + +// A Request created while discovery had nothing to offer keeps its spec through a later +// metadata-only update, even after the cluster recovers. +func TestUpdateKeepsHardwareUnresolved(t *testing.T) { + options := DGDRV1beta2Options{Hardware: &fakeDiscoverer{found: DiscoveredHardware{SKU: "h200_sxm", GPUs: 8}}} + // The objective metric is still defaulted on update, so the spec already carries it. + spec := v1beta2SchemaDefaulted() + metric := defaultMetricV1beta2 + spec.Objective.Metric = &metric + dgdr := v1beta2Envelope(t, spec) + first := string(dgdr.Spec.V1Beta2.Raw) + if err := defaultDynamoGraphDeploymentRequestV1beta2(context.Background(), dgdr, admissionv1.Update, options); err != nil { + t.Fatal(err) + } + if string(dgdr.Spec.V1Beta2.Raw) != first { + t.Fatalf("update changed the spec:\n%s\n%s", first, dgdr.Spec.V1Beta2.Raw) + } +} + +func TestDefaultSpecV1beta2StructuralDefaults(t *testing.T) { + opts := dynamoGraphDeploymentRequestV1beta2DefaultOptions{} + + t.Run("an empty native spec receives the four structural parents", func(t *testing.T) { + spec := &nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec{} + defaultDynamoGraphDeploymentRequestSpecV1beta2(spec, opts) + if len(spec.Backends) != 3 { + t.Fatalf("backends = %v, want vllm, sglang and trtllm", spec.Backends) + } + if !hasModelOnlyStatic(spec.Workload) { + t.Fatalf("workload = %s, want {} with the static model-only default", v1beta2JSON(t, spec)) + } + if spec.Objective == nil || spec.Recommendation == nil { + t.Fatalf("objective and recommendation must exist: %s", v1beta2JSON(t, spec)) + } + }) + t.Run("explicit values and opt-outs are kept", func(t *testing.T) { + spec := &nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec{} + spec.Backends = []nvidiacomv1beta2.Backend{nvidiacomv1beta2.BackendVLLM} + spec.Workload = &nvidiacomv1beta2.WorkloadSpec{Static: &nvidiacomv1beta2.StaticWorkloadSpec{}} + defaultDynamoGraphDeploymentRequestSpecV1beta2(spec, opts) + if len(spec.Backends) != 1 { + t.Fatalf("backends changed: %v", spec.Backends) + } + if spec.Workload.Static == nil { + t.Fatal("workload.static: {} (the opt-out) was removed") + } + }) +} + +func hasModelOnlyStatic(w *nvidiacomv1beta2.WorkloadSpec) bool { + return w != nil && w.Static != nil && + int32Is(w.Static.ISL, 1024) && int32Is(w.Static.OSL, 128) && int32Is(w.Static.Concurrency, 10) +} + +func int32Is(p *int32, want int32) bool { return p != nil && *p == want } diff --git a/deploy/operator/internal/webhook/request_version.go b/deploy/operator/internal/webhook/request_version.go new file mode 100644 index 000000000000..6bc43e1a5d5f --- /dev/null +++ b/deploy/operator/internal/webhook/request_version.go @@ -0,0 +1,48 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package webhook + +import "sigs.k8s.io/controller-runtime/pkg/webhook/admission" + +const nativeCarrierVersion = "v1beta2" + +// RequestedVersion is the API version the client used. With matchPolicy: Equivalent the API +// server converts the object to the version the webhook registered for; RequestKind then +// carries the original kind and Kind the converted one. RequestKind is absent when no +// conversion happened. +func RequestedVersion(req admission.Request) string { + if req.RequestKind != nil { + return req.RequestKind.Version + } + return req.Kind.Version +} + +// RequestUsesV1Beta2 reports whether the client addressed the Request through v1beta2. +func RequestUsesV1Beta2(req admission.Request) bool { + return RequestedVersion(req) == nativeCarrierVersion +} + +// RequestIsNative reports whether the represented spec is native to the API version the +// client used. hasV1Beta2Carrier is whether the converted v1beta1 object carries spec.v1beta2. +// After conversion that carrier means the represented object is native v1beta2, so the +// request is native exactly when both agree. A mismatch is a foreign-version view: the +// client either wrote a conversion carrier directly or reads or updates an object through +// the API version that does not own it. +func RequestIsNative(req admission.Request, hasV1Beta2Carrier bool) bool { + return RequestUsesV1Beta2(req) == hasV1Beta2Carrier +} diff --git a/deploy/operator/internal/webhook/request_version_test.go b/deploy/operator/internal/webhook/request_version_test.go new file mode 100644 index 000000000000..df630a49f1af --- /dev/null +++ b/deploy/operator/internal/webhook/request_version_test.go @@ -0,0 +1,78 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package webhook + +import ( + "testing" + + admissionv1 "k8s.io/api/admission/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" +) + +func requestWithKinds(kind string, requestKind string) admission.Request { + req := admission.Request{AdmissionRequest: admissionv1.AdmissionRequest{ + Kind: metav1.GroupVersionKind{Group: "nvidia.com", Version: kind, Kind: "DynamoGraphDeploymentRequest"}, + }} + if requestKind != "" { + req.RequestKind = &metav1.GroupVersionKind{Group: "nvidia.com", Version: requestKind, Kind: "DynamoGraphDeploymentRequest"} + } + return req +} + +func TestRequestedVersion(t *testing.T) { + tests := []struct { + name string + req admission.Request + want string + }{ + {"no conversion uses Kind", requestWithKinds("v1beta1", ""), "v1beta1"}, + {"RequestKind wins after conversion", requestWithKinds("v1beta1", "v1beta2"), "v1beta2"}, + {"RequestKind wins over a matching Kind", requestWithKinds("v1beta1", "v1alpha1"), "v1alpha1"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := RequestedVersion(tt.req); got != tt.want { + t.Fatalf("RequestedVersion() = %q, want %q", got, tt.want) + } + }) + } +} + +func TestRequestIsNative(t *testing.T) { + tests := []struct { + name string + req admission.Request + carrier bool + want bool + }{ + {"native v1beta2 create", requestWithKinds("v1beta1", "v1beta2"), true, true}, + {"native v1beta1 create", requestWithKinds("v1beta1", ""), false, true}, + {"native v1alpha1 create", requestWithKinds("v1beta1", "v1alpha1"), false, true}, + {"carrier written through v1beta1", requestWithKinds("v1beta1", ""), true, false}, + {"carrier written through v1alpha1", requestWithKinds("v1beta1", "v1alpha1"), true, false}, + {"v1beta1 payload written through v1beta2", requestWithKinds("v1beta1", "v1beta2"), false, false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := RequestIsNative(tt.req, tt.carrier); got != tt.want { + t.Fatalf("RequestIsNative() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/deploy/operator/internal/webhook/setup/setup.go b/deploy/operator/internal/webhook/setup/setup.go index 1da053a3207a..268fe4d312c5 100644 --- a/deploy/operator/internal/webhook/setup/setup.go +++ b/deploy/operator/internal/webhook/setup/setup.go @@ -23,10 +23,11 @@ type Options struct { Config *configv1alpha1.OperatorConfiguration RuntimeConfig *commoncontroller.RuntimeConfig OperatorVersion string - // DGDRDefaultImage is the default DGDR profiler image, put into - // DGDR spec.image when unset; empty derives - // dynamo-planner:. - DGDRDefaultImage string + // DGDRV1beta1 configures the defaults of v1beta1 Requests. + DGDRV1beta1 webhookdefaulting.DGDRV1beta1Options + // DGDRV1beta2 configures the defaults of v1beta2 Requests: the AISimulate runner + // image and the cluster hardware discovery. + DGDRV1beta2 webhookdefaulting.DGDRV1beta2Options OperatorPrincipal string Gate features.Gate } @@ -104,7 +105,8 @@ func Setup(mgr ctrl.Manager, opts Options) error { return fmt.Errorf("unable to register DynamoGraphDeployment defaulting webhook: %w", err) } - dgdrDefaulter := webhookdefaulting.NewDGDRDefaulter(opts.OperatorVersion, opts.DGDRDefaultImage) + dgdrDefaulter := webhookdefaulting.NewDGDRDefaulter(opts.OperatorVersion, opts.DGDRV1beta1.DefaultImage). + WithV1beta2(opts.DGDRV1beta2) if err := dgdrDefaulter.RegisterWithManager(mgr); err != nil { return fmt.Errorf("unable to register DynamoGraphDeploymentRequest defaulting webhook: %w", err) } diff --git a/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest.go b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest.go index 96cf6c0a80d3..dc31bcc88445 100644 --- a/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest.go +++ b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest.go @@ -27,7 +27,7 @@ import ( nvidiacomv1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" "github.com/ai-dynamo/dynamo/deploy/operator/internal/dgdoverride" "github.com/ai-dynamo/dynamo/deploy/operator/internal/features" - "github.com/ai-dynamo/dynamo/deploy/operator/internal/runtimeversion" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/versionoverride" apiequality "k8s.io/apimachinery/pkg/api/equality" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" k8sruntime "k8s.io/apimachinery/pkg/runtime" @@ -59,11 +59,6 @@ func (v *DynamoGraphDeploymentRequestValidator) Validate( ctx context.Context, request *nvidiacomv1beta1.DynamoGraphDeploymentRequest, ) (admission.Warnings, error) { - // Legacy v1beta1 rules do not apply to a converted v1beta2 carrier. - if request.Spec.V1Beta2 != nil { - return nil, nil - } - validation := &dynamoGraphDeploymentRequestValidation{ctx: ctx} allErrs := validation.validateDynamoGraphDeploymentRequest(request) return validation.warnings, invalidDynamoGraphDeploymentRequestError(request, allErrs) @@ -75,11 +70,6 @@ func (v *DynamoGraphDeploymentRequestValidator) ValidateUpdate( oldRequest *nvidiacomv1beta1.DynamoGraphDeploymentRequest, newRequest *nvidiacomv1beta1.DynamoGraphDeploymentRequest, ) (admission.Warnings, error) { - // Legacy v1beta1 rules do not apply to a converted v1beta2 carrier. - if newRequest.Spec.V1Beta2 != nil { - return nil, nil - } - validation := &dynamoGraphDeploymentRequestValidation{ctx: ctx} allErrs := validation.validateDynamoGraphDeploymentRequestUpdate(newRequest, oldRequest) validation.warnDeprecatedDGDOverrideTargets(&newRequest.Spec) @@ -99,6 +89,17 @@ func (v *dynamoGraphDeploymentRequestValidation) validateDynamoGraphDeploymentRe spec *nvidiacomv1beta1.DynamoGraphDeploymentRequestSpec, fldPath *field.Path, ) field.ErrorList { + // The requested API version decides what spec.v1beta2 means. A native request is + // validated; a spec that only conversion may produce is rejected on CREATE. + if !v.requestIsNative(spec.V1Beta2 != nil) { + return foreignCreateErrors(fldPath, spec.V1Beta2 != nil) + } + // A v1beta2 Request reaches this webhook as a v1beta1 object carrying spec.v1beta2; + // the v1beta1 rules below do not apply to it. + if spec.V1Beta2 != nil { + return validateDynamoGraphDeploymentRequestEnvelopeV1beta2(spec.V1Beta2, fldPath.Child("v1beta2")) + } + allErrs := field.ErrorList{} if !features.MustGateFrom(v.ctx).Enabled(features.GPUDiscovery) && !hasManualDGDRHardware(spec.Hardware) { @@ -251,6 +252,24 @@ func (v *dynamoGraphDeploymentRequestValidation) validateDynamoGraphDeploymentRe fldPath *field.Path, oldPhase nvidiacomv1beta1.DGDRPhase, ) field.ErrorList { + // A spec viewed through the API version that does not own it is read-only: an unchanged + // spec round-trips, any other write is rejected. + if !v.requestIsNative(newSpec.V1Beta2 != nil) { + if apiequality.Semantic.DeepEqual(newSpec, oldSpec) { + return nil + } + return foreignUpdateErrors(fldPath, newSpec.V1Beta2 != nil) + } + // A native v1beta2 spec is unconstrained by phase: an unchanged spec is always allowed, so + // status-preserving writes and defaulting-only updates pass, and any change is fully + // validated as if it were new. + if newSpec.V1Beta2 != nil { + if apiequality.Semantic.DeepEqual(newSpec, oldSpec) { + return nil + } + return validateDynamoGraphDeploymentRequestEnvelopeV1beta2(newSpec.V1Beta2, fldPath.Child("v1beta2")) + } + allErrs := field.ErrorList{} gpuDiscoveryEnabled := features.MustGateFrom(v.ctx).Enabled(features.GPUDiscovery) @@ -304,7 +323,7 @@ func dgdrRuntimeVersionOverrideRequired(spec *nvidiacomv1beta1.DynamoGraphDeploy if spec.Image == "" || spec.RuntimeVersionOverride != "" { return false } - _, err := runtimeversion.ParseImageVersion(spec.Image) + _, err := versionoverride.ParseImageVersion(spec.Image) return err != nil } diff --git a/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_carrier_envtest_test.go b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_carrier_envtest_test.go new file mode 100644 index 000000000000..2d44cc346ab5 --- /dev/null +++ b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_carrier_envtest_test.go @@ -0,0 +1,75 @@ +package validation_test + +import ( + "sort" + "testing" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/dynamic" + + "github.com/ai-dynamo/dynamo/deploy/operator/internal/features" +) + +// A request created through its owning API version has to stay writable through every other +// served version. A metadata-only patch through the foreign view must not be rejected, and it +// must neither rewrite the spec nor bump the generation. A default that materialises a native +// field next to a conversion carrier breaks this, so it is covered at the API-server boundary. +func TestDynamoGraphDeploymentRequestCarrierObjectsStayWritable(t *testing.T) { + t.Run("native v1beta1 written through v1beta2", func(t *testing.T) { + carrierRoundTrip(t, betaDGDRForAdmission(nil), beta2DGDRForAdmission(nil), "v1beta1") + }) + t.Run("native v1beta2 written through v1beta1", func(t *testing.T) { + carrierRoundTrip(t, beta2DGDRForAdmission(nil), betaDGDRForAdmission(nil), "v1beta2") + }) +} + +func carrierRoundTrip(t *testing.T, ownerFixture, otherFixture runtime.Object, carrierKey string) { + t.Helper() + env := admissionEnv.ForTest(t) + admissionGate.set(features.Gates{GPUDiscovery: true}) + warnings := &warningRecorder{} + ownerClient := newAdmissionResourceClient(t, env, ownerFixture, "", warnings) + otherClient := newAdmissionResourceClient(t, env, otherFixture, "", warnings) + + t.Log("Create the request through the version that owns its representation") + object, _ := admissionObject(t, ownerFixture, env.Namespace(), nil) + created, err := ownerClient.Create(t.Context(), object, metav1.CreateOptions{}) + if err != nil { + t.Fatalf("create through the owning version: %v", err) + } + name := created.GetName() + + t.Log("Read it through the other version and expect the conversion carrier") + foreign, err := otherClient.Get(t.Context(), name, metav1.GetOptions{}) + if err != nil { + t.Fatalf("read through the other version: %v", err) + } + spec, _ := foreign.Object["spec"].(map[string]any) + if _, ok := spec[carrierKey]; !ok { + keys := make([]string, 0, len(spec)) + for key := range spec { + keys = append(keys, key) + } + sort.Strings(keys) + t.Fatalf("foreign view has spec keys %v, want the %s carrier", keys, carrierKey) + } + + t.Log("Patch metadata through each version") + for label, client := range map[string]dynamic.ResourceInterface{"foreign": otherClient, "owner": ownerClient} { + patch := []byte(`{"metadata":{"labels":{"carrier-probe-` + label + `":"1"}}}`) + if _, err := client.Patch(t.Context(), name, types.MergePatchType, patch, metav1.PatchOptions{}); err != nil { + t.Fatalf("metadata-only patch through the %s version: %v", label, err) + } + } + + t.Log("The spec and generation must be untouched") + after, err := ownerClient.Get(t.Context(), name, metav1.GetOptions{}) + if err != nil { + t.Fatalf("read back through the owning version: %v", err) + } + if after.GetGeneration() != created.GetGeneration() { + t.Fatalf("generation = %d, want %d: a metadata-only patch rewrote the spec", after.GetGeneration(), created.GetGeneration()) + } +} diff --git a/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_routing.go b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_routing.go new file mode 100644 index 000000000000..71a6419cdd8c --- /dev/null +++ b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_routing.go @@ -0,0 +1,68 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package validation + +import ( + internalwebhook "github.com/ai-dynamo/dynamo/deploy/operator/internal/webhook" + "k8s.io/apimachinery/pkg/util/validation/field" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" +) + +// requestIsNative reports whether the represented spec is native to the API version the +// client used (see internalwebhook.RequestIsNative). hasV1Beta2Carrier is whether the +// converted v1beta1 object carries spec.v1beta2. Without an admission request in the +// context (direct calls from unit tests) there is no foreign view, so the request counts as +// native. +func (v *dynamoGraphDeploymentRequestValidation) requestIsNative(hasV1Beta2Carrier bool) bool { + req, err := admission.RequestFromContext(v.ctx) + if err != nil { + return true + } + return internalwebhook.RequestIsNative(req, hasV1Beta2Carrier) +} + +// foreignCreateErrors rejects a spec that conversion alone may produce. The carrier is raw +// JSON that bypasses structural pruning; it is safe only when conversion built it from a +// native request that was validated against its own versioned schema. +func foreignCreateErrors(fldPath *field.Path, hasV1Beta2Carrier bool) field.ErrorList { + if hasV1Beta2Carrier { + return field.ErrorList{field.Forbidden( + fldPath.Child("v1beta2"), + "spec.v1beta2 is a conversion carrier that only conversion may write; create the Request through the v1beta2 API", + )} + } + return field.ErrorList{field.Forbidden( + fldPath, + "a v1beta1 representation cannot be written through the v1beta2 API; spec.v1beta1 is a conversion carrier that only conversion may write", + )} +} + +// foreignUpdateErrors rejects a change to a spec viewed through the API version that does +// not own it. An unchanged spec never reaches this function. +func foreignUpdateErrors(fldPath *field.Path, hasV1Beta2Carrier bool) field.ErrorList { + if hasV1Beta2Carrier { + return field.ErrorList{field.Forbidden( + fldPath.Child("v1beta2"), + "spec.v1beta2 is a read-only view through this API version; update the Request through the v1beta2 API", + )} + } + return field.ErrorList{field.Forbidden( + fldPath, + "this Request is owned by v1beta1; its spec is read-only through the v1beta2 API, update it through v1beta1", + )} +} diff --git a/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_v1beta2.go b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_v1beta2.go new file mode 100644 index 000000000000..ab5799d779d8 --- /dev/null +++ b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_v1beta2.go @@ -0,0 +1,125 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package validation + +import ( + "encoding/json" + "strings" + + nvidiacomv1beta2 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta2" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/controller/aisimulate" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + "k8s.io/apimachinery/pkg/util/validation/field" +) + +// parametersPath is where the native AISimulate input lives in a v1beta2 Request. +var parametersPath = field.NewPath("spec", "aisimulate", "parameters") + +// validateDynamoGraphDeploymentRequestEnvelopeV1beta2 validates the v1beta2 spec carried in +// spec.v1beta2 of a v1beta1 object. raw is that field and fldPath is its path; a spec that +// does not decode is reported there. Errors about the decoded spec use the v1beta2 spec's own +// field paths (spec.model, spec.aisimulate, ...), which are what the user wrote, so they are +// rooted at spec rather than at fldPath. +func validateDynamoGraphDeploymentRequestEnvelopeV1beta2( + raw *apiextensionsv1.JSON, + fldPath *field.Path, +) field.ErrorList { + var spec nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec + if err := json.Unmarshal(raw.Raw, &spec); err != nil { + return field.ErrorList{field.Invalid(fldPath, "", "must be a valid v1beta2 spec: "+err.Error())} + } + return validateDynamoGraphDeploymentRequestV1beta2(&spec) +} + +// validateDynamoGraphDeploymentRequestV1beta2 validates a defaulted v1beta2 Request spec: the +// typed fields, then that spec.aisimulate.parameters is an object. The content of the +// parameters is not validated here; the AISimulate input schema owns it. +func validateDynamoGraphDeploymentRequestV1beta2(spec *nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec) field.ErrorList { + errs := validateDynamoGraphDeploymentRequestSpecV1beta2(spec, field.NewPath("spec")) + if _, err := aisimulate.Parameters(spec); err != nil { + errs = append(errs, field.Invalid(parametersPath, "", err.Error())) + } + return errs +} + +// validateDynamoGraphDeploymentRequestSpecV1beta2 validates spec. spec and fldPath must not be nil. +func validateDynamoGraphDeploymentRequestSpecV1beta2( + spec *nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec, + fldPath *field.Path, +) field.ErrorList { + var errs field.ErrorList + errs = append(errs, validateModelSpecV1beta2(spec.Model, fldPath.Child("model"))...) + errs = append(errs, validateBackendsV1beta2(spec.Backends, fldPath.Child("backends"))...) + if spec.Workload != nil { + errs = append(errs, validateWorkloadSpecV1beta2(spec.Workload, fldPath.Child("workload"))...) + } + errs = append(errs, validateAISimulateSpecV1beta2(spec.AISimulate, fldPath.Child("aisimulate"))...) + return errs +} + +func validateModelSpecV1beta2(model *nvidiacomv1beta2.ModelSpec, fldPath *field.Path) field.ErrorList { + if model == nil { + return field.ErrorList{field.Required(fldPath, "model is required")} + } + if strings.TrimSpace(model.Name) == "" { + return field.ErrorList{field.Required(fldPath.Child("name"), "must not be blank")} + } + return nil +} + +func validateBackendsV1beta2(backends []nvidiacomv1beta2.Backend, fldPath *field.Path) field.ErrorList { + var errs field.ErrorList + seen := make(map[nvidiacomv1beta2.Backend]bool, len(backends)) + for i, backend := range backends { + if seen[backend] { + errs = append(errs, field.Duplicate(fldPath.Index(i), backend)) + } + seen[backend] = true + } + return errs +} + +func validateWorkloadSpecV1beta2(workload *nvidiacomv1beta2.WorkloadSpec, fldPath *field.Path) field.ErrorList { + if workload.Static == nil { + return nil + } + return validateStaticWorkloadSpecV1beta2(workload.Static, fldPath.Child("static")) +} + +func validateStaticWorkloadSpecV1beta2(static *nvidiacomv1beta2.StaticWorkloadSpec, fldPath *field.Path) field.ErrorList { + // The schema bounds each ratio to [0, 1] and the number of entries to one or two. + if len(static.KVLoadRatio) == 2 && static.KVLoadRatio[0] >= static.KVLoadRatio[1] { + return field.ErrorList{field.Invalid(fldPath.Child("kvLoadRatio"), static.KVLoadRatio, "a range needs min < max")} + } + return nil +} + +// validateAISimulateSpecV1beta2 validates the execution artifact. Admission persists the +// operator's default image, so a Request that still has none was created while the operator +// had no default image configured. spec may be nil. +func validateAISimulateSpecV1beta2(spec *nvidiacomv1beta2.AISimulateSpec, fldPath *field.Path) field.ErrorList { + if spec == nil || spec.Image == "" { + return field.ErrorList{field.Required(fldPath.Child("image"), + "an AISimulate image is required; set it, or configure the operator's default AISimulate image")} + } + if _, err := aisimulate.ResolveVersion(spec.Image, spec.VersionOverride); err != nil { + return field.ErrorList{field.Required(fldPath.Child("versionOverride"), + "is required when spec.aisimulate.image has no parseable version tag")} + } + return nil +} diff --git a/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_v1beta2_test.go b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_v1beta2_test.go new file mode 100644 index 000000000000..e008602955b5 --- /dev/null +++ b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_v1beta2_test.go @@ -0,0 +1,181 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package validation + +import ( + "encoding/json" + "strings" + "testing" + + nvidiacomv1beta2 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta2" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/validation/field" +) + +func v1beta2Ptr[T any](v T) *T { return &v } + +// v1beta2DefaultedSpec is a Request as it looks once the schema and admission have defaulted it. +func v1beta2DefaultedSpec() *nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec { + metric := nvidiacomv1beta2.ObjectiveMetric("ThroughputPerGPU") + return &nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec{ + Model: &nvidiacomv1beta2.ModelSpec{Name: "meta-llama/Llama-3.1-8B-Instruct"}, + Backends: []nvidiacomv1beta2.Backend{nvidiacomv1beta2.BackendVLLM, nvidiacomv1beta2.BackendSGLang, nvidiacomv1beta2.BackendTRTLLM}, + Hardware: &nvidiacomv1beta2.HardwareSpec{GPU: &nvidiacomv1beta2.GPUHardwareSpec{SKU: v1beta2Ptr("h200_sxm"), Budget: v1beta2Ptr(int32(8))}}, + Workload: &nvidiacomv1beta2.WorkloadSpec{Static: &nvidiacomv1beta2.StaticWorkloadSpec{ + ISL: v1beta2Ptr(int32(4000)), OSL: v1beta2Ptr(int32(1000)), KVLoadRatio: nvidiacomv1beta2.KVLoadRatio{0, 1}, + }}, + Objective: &nvidiacomv1beta2.ObjectiveSpec{ + Mode: nvidiacomv1beta2.ObjectiveModeOptimize, + Metric: &metric, + SLA: &nvidiacomv1beta2.ObjectiveSLASpec{TTFTMs: v1beta2Ptr(2000.0), ITLMs: v1beta2Ptr(30.0)}, + }, + AISimulate: &nvidiacomv1beta2.AISimulateSpec{Image: "registry.example.com/aisimulate:1.2.0"}, + Recommendation: &nvidiacomv1beta2.RecommendationSpec{MaxCandidates: 1}, + } +} + +func v1beta2WithParameters(spec *nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec, parameters string) *nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec { + if spec.AISimulate == nil { + spec.AISimulate = &nvidiacomv1beta2.AISimulateSpec{Image: "registry.example.com/aisimulate:1.2.0"} + } + spec.AISimulate.Parameters = &runtime.RawExtension{Raw: []byte(parameters)} + return spec +} + +func v1beta2Details(errs field.ErrorList) string { + var out []string + for _, err := range errs { + out = append(out, err.Field+": "+err.Detail) + } + return strings.Join(out, "\n") +} + +func v1beta2RequireError(t *testing.T, errs field.ErrorList, substr string) { + t.Helper() + if !strings.Contains(v1beta2Details(errs), substr) { + t.Fatalf("want an error containing %q, got:\n%s", substr, v1beta2Details(errs)) + } +} + +func v1beta2RequireNoError(t *testing.T, errs field.ErrorList) { + t.Helper() + if len(errs) > 0 { + t.Fatalf("unexpected errors:\n%s", v1beta2Details(errs)) + } +} + +func TestValidateV1beta2AcceptsADefaultedRequest(t *testing.T) { + v1beta2RequireNoError(t, validateDynamoGraphDeploymentRequestV1beta2(v1beta2DefaultedSpec())) +} + +func TestValidateV1beta2TypedStage(t *testing.T) { + t.Run("missing model", func(t *testing.T) { + spec := v1beta2DefaultedSpec() + spec.Model = nil + errs := validateDynamoGraphDeploymentRequestV1beta2(spec) + v1beta2RequireError(t, errs, "spec.model: model is required") + }) + t.Run("blank model", func(t *testing.T) { + spec := v1beta2DefaultedSpec() + spec.Model.Name = " " + errs := validateDynamoGraphDeploymentRequestV1beta2(spec) + v1beta2RequireError(t, errs, "spec.model.name: must not be blank") + }) + t.Run("duplicate backend", func(t *testing.T) { + spec := v1beta2DefaultedSpec() + spec.Backends = []nvidiacomv1beta2.Backend{nvidiacomv1beta2.BackendVLLM, nvidiacomv1beta2.BackendVLLM} + v1beta2RequireError(t, validateDynamoGraphDeploymentRequestV1beta2(spec), "spec.backends[1]") + }) + t.Run("kv_load_ratio range order", func(t *testing.T) { + spec := v1beta2DefaultedSpec() + spec.Workload.Static.KVLoadRatio = nvidiacomv1beta2.KVLoadRatio{0.9, 0.1} + v1beta2RequireError(t, validateDynamoGraphDeploymentRequestV1beta2(spec), "spec.workload.static.kvLoadRatio: a range needs min < max") + }) + t.Run("an empty workload has nothing to validate", func(t *testing.T) { + spec := v1beta2WithParameters(v1beta2DefaultedSpec(), `{"workload":{"isl":512,"osl":64,"concurrency":8}}`) + spec.Workload = &nvidiacomv1beta2.WorkloadSpec{} + v1beta2RequireNoError(t, validateDynamoGraphDeploymentRequestV1beta2(spec)) + }) + t.Run("an invalid typed value is not excused by the parameters", func(t *testing.T) { + spec := v1beta2WithParameters(v1beta2DefaultedSpec(), `{"workload":{"kv_load_ratio":0.5}}`) + spec.Workload.Static.KVLoadRatio = nvidiacomv1beta2.KVLoadRatio{0.9, 0.1} + v1beta2RequireError(t, validateDynamoGraphDeploymentRequestV1beta2(spec), "spec.workload.static.kvLoadRatio") + }) + t.Run("typed and parameter errors aggregate", func(t *testing.T) { + spec := v1beta2WithParameters(v1beta2DefaultedSpec(), `[1]`) + spec.Model.Name = "" + errs := validateDynamoGraphDeploymentRequestV1beta2(spec) + v1beta2RequireError(t, errs, "spec.model.name") + v1beta2RequireError(t, errs, "spec.aisimulate.parameters") + }) + t.Run("parameter content is not validated here", func(t *testing.T) { + v1beta2RequireNoError(t, validateDynamoGraphDeploymentRequestV1beta2( + v1beta2WithParameters(v1beta2DefaultedSpec(), `{"workload":{"isl":0},"goal":{"target":"nope"}}`))) + }) +} + +func TestValidateV1beta2AISimulateImage(t *testing.T) { + t.Run("no AISimulate section", func(t *testing.T) { + spec := v1beta2DefaultedSpec() + spec.AISimulate = nil + v1beta2RequireError(t, validateDynamoGraphDeploymentRequestV1beta2(spec), "spec.aisimulate.image: an AISimulate image is required") + }) + t.Run("parameters without an image", func(t *testing.T) { + spec := v1beta2DefaultedSpec() + spec.AISimulate = &nvidiacomv1beta2.AISimulateSpec{} + v1beta2RequireError(t, validateDynamoGraphDeploymentRequestV1beta2(spec), "spec.aisimulate.image") + }) + t.Run("custom tag needs a version", func(t *testing.T) { + spec := v1beta2DefaultedSpec() + spec.AISimulate.Image = "registry.example.com/aisimulate:custom" + v1beta2RequireError(t, validateDynamoGraphDeploymentRequestV1beta2(spec), "spec.aisimulate.versionOverride: is required when spec.aisimulate.image has no parseable version tag") + }) + t.Run("custom tag with a version", func(t *testing.T) { + spec := v1beta2DefaultedSpec() + spec.AISimulate.Image = "registry.example.com/aisimulate:custom" + spec.AISimulate.VersionOverride = "1.4.0" + v1beta2RequireNoError(t, validateDynamoGraphDeploymentRequestV1beta2(spec)) + }) +} + +func TestValidateV1beta2MalformedParameters(t *testing.T) { + errs := validateDynamoGraphDeploymentRequestV1beta2(v1beta2WithParameters(v1beta2DefaultedSpec(), `[1]`)) + v1beta2RequireError(t, errs, "spec.aisimulate.parameters: spec.aisimulate.parameters: must be a JSON object") +} + +func v1beta2Envelope(raw string) *apiextensionsv1.JSON { + return &apiextensionsv1.JSON{Raw: []byte(raw)} +} + +func TestValidateEnvelopeV1beta2(t *testing.T) { + path := field.NewPath("spec", "v1beta2") + + good, err := json.Marshal(v1beta2DefaultedSpec()) + if err != nil { + t.Fatal(err) + } + v1beta2RequireNoError(t, validateDynamoGraphDeploymentRequestEnvelopeV1beta2(v1beta2Envelope(string(good)), path)) + + bad := v1beta2DefaultedSpec() + bad.Model.Name = "" + raw, _ := json.Marshal(bad) + v1beta2RequireError(t, validateDynamoGraphDeploymentRequestEnvelopeV1beta2(v1beta2Envelope(string(raw)), path), "model") + + v1beta2RequireError(t, validateDynamoGraphDeploymentRequestEnvelopeV1beta2(v1beta2Envelope(`{"model": 3}`), path), "spec.v1beta2: must be a valid v1beta2 spec") +} diff --git a/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_validation_envtest_test.go b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_validation_envtest_test.go index 0aeca354f295..29055c68eaf9 100644 --- a/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_validation_envtest_test.go +++ b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_validation_envtest_test.go @@ -18,6 +18,9 @@ package validation_test import ( + "encoding/json" + "errors" + "reflect" "strings" "testing" @@ -26,6 +29,7 @@ import ( nvidiacomv1beta2 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta2" "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" "github.com/ai-dynamo/dynamo/deploy/operator/internal/features" + webhookdefaulting "github.com/ai-dynamo/dynamo/deploy/operator/internal/webhook/defaulting" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" @@ -56,18 +60,27 @@ func TestDynamoGraphDeploymentRequestValidator_Validate(t *testing.T) { "spec":{"components":[{"name":"VllmPrefillWorker"}]} }` + discoveredH200 := webhookdefaulting.DiscoveredHardware{SKU: "h200_sxm", GPUs: 8} + tests := []struct { name string request runtime.Object oldRequest runtime.Object gpuDiscovery bool seedWithoutWebhook bool - wantImage string - wantSchemaErr string - wantCELErr string - wantStatusErr string - wantWebhook []string - wantWarnings []string + // discovered and discoveryErr are what cluster GPU discovery reports while the case + // runs; beforeUpdate may change them between the seed and the update. + discovered webhookdefaulting.DiscoveredHardware + discoveryErr error + beforeUpdate func(*testing.T) + // check inspects the object the API server returned. + check func(*testing.T, *unstructured.Unstructured) + wantImage string + wantSchemaErr string + wantCELErr string + wantStatusErr string + wantWebhook []string + wantWarnings []string }{ // Source-version schema, CEL, and conversion boundaries. { @@ -532,10 +545,239 @@ func TestDynamoGraphDeploymentRequestValidator_Validate(t *testing.T) { `spec.searchStrategy: Invalid value: "thorough": is incompatible with spec.backend "auto"; set spec.backend to a specific backend (sglang, trtllm, or vllm)`, }, }, + + // v1beta2: the CRD owns the unconditional defaults, admission the environment ones. + { + name: "v1beta2 model-only request gets schema and admission defaults", + request: v1beta2DGDRForAdmission(nil, nil), + gpuDiscovery: true, + discovered: discoveredH200, + check: func(t *testing.T, result *unstructured.Unstructured) { + expectV1beta2SchemaDefaults(t, result) + expectV1beta2AdmissionDefaults(t, result) + }, + }, + { + name: "v1beta2 explicit hardware is kept", + request: v1beta2DGDRForAdmission(func(spec map[string]any) { + spec["hardware"] = map[string]any{"gpu": map[string]any{"sku": "h100_sxm", "budget": int64(4)}} + }, nil), + gpuDiscovery: true, + discovered: discoveredH200, + check: func(t *testing.T, result *unstructured.Unstructured) { + if got := v1beta2String(t, result, "spec", "hardware", "gpu", "sku"); got != "h100_sxm" { + t.Fatalf("hardware.gpu.sku = %q, want h100_sxm", got) + } + if got := v1beta2Number(t, result, "spec", "hardware", "gpu", "budget"); got != 4 { + t.Fatalf("hardware.gpu.budget = %v, want 4", got) + } + }, + }, + { + name: "v1beta2 failed discovery leaves the hardware absent", + request: v1beta2DGDRForAdmission(nil, nil), + gpuDiscovery: true, + discoveryErr: errors.New("no GPUs found"), + check: func(t *testing.T, result *unstructured.Unstructured) { + v1beta2Absent(t, result, "spec", "hardware") + if got := v1beta2String(t, result, "spec", "aisimulate", "image"); got != admissionAISimulateImage { + t.Fatalf("aisimulate.image = %q, want %q", got, admissionAISimulateImage) + } + }, + }, + { + name: "v1beta2 empty workload receives the static model-only default", + request: v1beta2DGDRForAdmission(func(spec map[string]any) { + spec["workload"] = map[string]any{} + }, nil), + gpuDiscovery: true, + discovered: discoveredH200, + check: func(t *testing.T, result *unstructured.Unstructured) { + for key, want := range map[string]float64{"isl": 1024, "osl": 128, "concurrency": 10} { + if got := v1beta2Number(t, result, "spec", "workload", "static", key); got != want { + t.Fatalf("workload.static.%s = %v, want %v", key, got, want) + } + } + v1beta2Absent(t, result, "spec", "workload", "static", "kvLoadRatio") + }, + }, + { + name: "v1beta2 empty workload static is an opt-out and stays empty", + request: v1beta2DGDRForAdmission(func(spec map[string]any) { + spec["workload"] = map[string]any{"static": map[string]any{}} + }, nil), + gpuDiscovery: true, + discovered: discoveredH200, + check: func(t *testing.T, result *unstructured.Unstructured) { + v1beta2Empty(t, result, "spec", "workload", "static") + }, + }, + { + name: "v1beta2 empty workload static stays empty beside native parameters", + request: v1beta2DGDRForAdmission(func(spec map[string]any) { + spec["workload"] = map[string]any{"static": map[string]any{}} + spec["aisimulate"] = map[string]any{"parameters": mustJSONObject(t, `{"workload":{"isl":512,"osl":128,"concurrency":8,"request_count":100}}`)} + }, nil), + gpuDiscovery: true, + discovered: discoveredH200, + check: func(t *testing.T, result *unstructured.Unstructured) { + v1beta2Empty(t, result, "spec", "workload", "static") + v1beta2Absent(t, result, "spec", "objective", "sla") + expectV1beta2AdmissionDefaults(t, result) + }, + }, + { + name: "v1beta2 empty sla is an opt-out and is not refilled", + request: v1beta2DGDRForAdmission(func(spec map[string]any) { + spec["objective"] = map[string]any{"sla": map[string]any{}} + }, nil), + gpuDiscovery: true, + discovered: discoveredH200, + check: func(t *testing.T, result *unstructured.Unstructured) { + v1beta2Empty(t, result, "spec", "objective", "sla") + if got := v1beta2Number(t, result, "spec", "workload", "static", "isl"); got != 1024 { + t.Fatalf("workload.static.isl = %v, want the schema default 1024", got) + } + }, + }, + { + name: "v1beta2 native parameters win without rewriting the typed spec", + request: v1beta2DGDRForAdmission(func(spec map[string]any) { + spec["aisimulate"] = map[string]any{"parameters": mustJSONObject(t, `{"workload":{"isl":256}}`)} + }, nil), + gpuDiscovery: true, + discovered: discoveredH200, + check: func(t *testing.T, result *unstructured.Unstructured) { + if got := v1beta2Number(t, result, "spec", "workload", "static", "isl"); got != 1024 { + t.Fatalf("typed workload.static.isl = %v, want 1024", got) + } + if got := v1beta2Number(t, result, "spec", "aisimulate", "parameters", "workload", "isl"); got != 256 { + t.Fatalf("parameters.workload.isl = %v, want 256", got) + } + }, + }, + { + name: "v1beta2 duplicate backends are rejected on create", + request: v1beta2DGDRForAdmission(func(spec map[string]any) { + spec["backends"] = []any{"vllm", "vllm"} + }, nil), + gpuDiscovery: true, + discovered: discoveredH200, + wantWebhook: []string{`spec.backends[1]: Duplicate value: "vllm"`}, + }, + { + name: "v1beta2 versionOverride latest is rejected by the schema", + request: v1beta2DGDRForAdmission(func(spec map[string]any) { + spec["aisimulate"] = map[string]any{"versionOverride": "latest"} + }, nil), + gpuDiscovery: true, + discovered: discoveredH200, + wantSchemaErr: `spec.aisimulate.versionOverride: Invalid value: "latest": spec.aisimulate.versionOverride in body should match '^(0|[1-9][0-9]{0,3})\.(0|[1-9][0-9]{0,3})\.(0|[1-9][0-9]{0,3})$'`, + }, + { + name: "v1beta2 versionOverride with a v prefix is rejected by the schema", + request: v1beta2DGDRForAdmission(func(spec map[string]any) { + spec["aisimulate"] = map[string]any{"versionOverride": "v0.13.0"} + }, nil), + gpuDiscovery: true, + discovered: discoveredH200, + wantSchemaErr: `spec.aisimulate.versionOverride: Invalid value: "v0.13.0": spec.aisimulate.versionOverride in body should match '^(0|[1-9][0-9]{0,3})\.(0|[1-9][0-9]{0,3})\.(0|[1-9][0-9]{0,3})$'`, + }, + { + name: "v1beta2 versionOverride 0.13.0 is accepted", + request: v1beta2DGDRForAdmission(func(spec map[string]any) { + spec["aisimulate"] = map[string]any{"versionOverride": "0.13.0"} + }, nil), + gpuDiscovery: true, + discovered: discoveredH200, + check: func(t *testing.T, result *unstructured.Unstructured) { + if got := v1beta2String(t, result, "spec", "aisimulate", "versionOverride"); got != "0.13.0" { + t.Fatalf("versionOverride = %q, want 0.13.0", got) + } + }, + }, + + // v1beta2 UPDATE: defaults stay idempotent, hardware is never rediscovered, and any + // spec change is fully validated. + { + name: "v1beta2 update without a spec change keeps the spec and generation", + oldRequest: v1beta2DGDRForAdmission(nil, nil), + request: v1beta2DGDRForAdmission(nil, map[string]any{"updated": "true"}), + gpuDiscovery: true, + discovered: discoveredH200, + check: func(t *testing.T, result *unstructured.Unstructured) { + if result.GetGeneration() != 1 { + t.Fatalf("generation = %d, want 1: defaulting must not change an unchanged spec", result.GetGeneration()) + } + expectV1beta2SchemaDefaults(t, result) + expectV1beta2AdmissionDefaults(t, result) + }, + }, + { + name: "v1beta2 metadata-only update does not pick up hardware once discovery recovers", + oldRequest: v1beta2DGDRForAdmission(nil, nil), + request: v1beta2DGDRForAdmission(nil, map[string]any{"updated": "true"}), + gpuDiscovery: true, + discoveryErr: errors.New("no GPUs found"), + beforeUpdate: func(*testing.T) { admissionHardware.set(discoveredH200, nil) }, + check: func(t *testing.T, result *unstructured.Unstructured) { + if result.GetGeneration() != 1 { + t.Fatalf("generation = %d, want 1: an update must not rewrite the spec", result.GetGeneration()) + } + v1beta2Absent(t, result, "spec", "hardware") + }, + }, + { + name: "v1beta2 native-only hardware survives a metadata-only update after discovery recovers", + oldRequest: v1beta2NativeHardwareDGDR(nil), + request: v1beta2NativeHardwareDGDR(map[string]any{"updated": "true"}), + gpuDiscovery: true, + discoveryErr: errors.New("no GPUs found"), + beforeUpdate: func(*testing.T) { admissionHardware.set(discoveredH200, nil) }, + check: func(t *testing.T, result *unstructured.Unstructured) { + if result.GetGeneration() != 1 { + t.Fatalf("generation = %d, want 1", result.GetGeneration()) + } + v1beta2Absent(t, result, "spec", "hardware") + if got := v1beta2String(t, result, "spec", "aisimulate", "parameters", "search_space", "hardware_sku"); got != "h100_sxm" { + t.Fatalf("parameters.search_space.hardware_sku = %q, want h100_sxm", got) + } + }, + }, + { + name: "v1beta2 spec change is accepted and bumps the generation", + oldRequest: v1beta2DGDRForAdmission(nil, nil), + request: v1beta2DGDRForAdmission(func(spec map[string]any) { + spec["model"] = map[string]any{"name": alternateAdmissionModel} + }, nil), + gpuDiscovery: true, + discovered: discoveredH200, + check: func(t *testing.T, result *unstructured.Unstructured) { + if result.GetGeneration() != 2 { + t.Fatalf("generation = %d, want 2", result.GetGeneration()) + } + if got := v1beta2String(t, result, "spec", "model", "name"); got != alternateAdmissionModel { + t.Fatalf("model.name = %q, want %q", got, alternateAdmissionModel) + } + }, + }, + { + name: "v1beta2 update to duplicate backends is rejected", + oldRequest: v1beta2DGDRForAdmission(nil, nil), + request: v1beta2DGDRForAdmission(func(spec map[string]any) { + spec["backends"] = []any{"sglang", "sglang"} + }, nil), + gpuDiscovery: true, + discovered: discoveredH200, + wantWebhook: []string{`spec.backends[1]: Duplicate value: "sglang"`}, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { + admissionHardware.set(tt.discovered, tt.discoveryErr) + t.Cleanup(func() { admissionHardware.set(webhookdefaulting.DiscoveredHardware{}, nil) }) + gates := features.Gates{GPUDiscovery: tt.gpuDiscovery} test := admissionTestCase{ object: tt.request, @@ -547,6 +789,7 @@ func TestDynamoGraphDeploymentRequestValidator_Validate(t *testing.T) { wantStatusError: tt.wantStatusErr, wantWebhookErrors: tt.wantWebhook, wantWarnings: tt.wantWarnings, + beforeUpdate: tt.beforeUpdate, } if tt.oldRequest != nil && !tt.gpuDiscovery { @@ -555,6 +798,9 @@ func TestDynamoGraphDeploymentRequestValidator_Validate(t *testing.T) { test.seedGates = &seedGates } actual := runAdmissionTest(t, test) + if tt.check != nil { + tt.check(t, actual) + } if tt.wantImage != "" { image, found, err := unstructured.NestedString(actual.Object, "spec", "image") if err != nil || !found { @@ -764,3 +1010,134 @@ func beta2DGDRForAdmission( } return request } + +const v1beta2AdmissionModel = "Qwen/Qwen3-0.6B" + +// v1beta2NativeHardwareDGDR supplies hardware only through native AISimulate parameters. +func v1beta2NativeHardwareDGDR(labels map[string]any) *unstructured.Unstructured { + return v1beta2DGDRForAdmission(func(spec map[string]any) { + spec["aisimulate"] = map[string]any{ + "image": admissionAISimulateImage, + "parameters": map[string]any{ + "search_space": map[string]any{"hardware_sku": "h100_sxm", "gpu_budget": int64(8)}, + }, + } + }, labels) +} + +// v1beta2DGDRForAdmission builds a v1beta2 Request as a client sends it, before the CRD +// defaults. It is unstructured on purpose: an omitted field, +// an explicit empty object and a null are different requests, and typed Go cannot tell them +// apart. +func v1beta2DGDRForAdmission(mutate func(spec map[string]any), labels map[string]any) *unstructured.Unstructured { + spec := map[string]any{"model": map[string]any{"name": v1beta2AdmissionModel}} + if mutate != nil { + mutate(spec) + } + metadata := map[string]any{"name": "test-v1beta2-dgdr", "namespace": "default"} + if labels != nil { + metadata["labels"] = labels + } + return &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": "nvidia.com/v1beta2", + "kind": "DynamoGraphDeploymentRequest", + "metadata": metadata, + "spec": spec, + }} +} + +// mustJSONObject decodes a native AISimulate parameters document. +func mustJSONObject(t *testing.T, document string) map[string]any { + t.Helper() + var value map[string]any + if err := json.Unmarshal([]byte(document), &value); err != nil { + t.Fatalf("decode %q: %v", document, err) + } + return value +} + +func v1beta2Field(t *testing.T, obj *unstructured.Unstructured, path ...string) (any, bool) { + t.Helper() + value, found, err := unstructured.NestedFieldNoCopy(obj.Object, path...) + if err != nil { + t.Fatalf("read %v: %v", path, err) + } + return value, found +} + +func v1beta2Number(t *testing.T, obj *unstructured.Unstructured, path ...string) float64 { + t.Helper() + value, found := v1beta2Field(t, obj, path...) + if !found { + t.Fatalf("%v is not set", path) + } + switch n := value.(type) { + case int64: + return float64(n) + case float64: + return n + default: + t.Fatalf("%v = %#v, want a number", path, value) + return 0 + } +} + +func v1beta2String(t *testing.T, obj *unstructured.Unstructured, path ...string) string { + t.Helper() + value, found := v1beta2Field(t, obj, path...) + if !found { + t.Fatalf("%v is not set", path) + } + text, ok := value.(string) + if !ok { + t.Fatalf("%v = %#v, want a string", path, value) + } + return text +} + +func v1beta2Absent(t *testing.T, obj *unstructured.Unstructured, path ...string) { + t.Helper() + if value, found := v1beta2Field(t, obj, path...); found { + t.Fatalf("%v = %#v, want it absent", path, value) + } +} + +func v1beta2Empty(t *testing.T, obj *unstructured.Unstructured, path ...string) { + t.Helper() + value, found := v1beta2Field(t, obj, path...) + if !found || !reflect.DeepEqual(value, map[string]any{}) { + t.Fatalf("%v = %#v (found=%v), want an empty object", path, value, found) + } +} + +// expectV1beta2SchemaDefaults checks what the CRD supplies when workload and sla are omitted. +func expectV1beta2SchemaDefaults(t *testing.T, obj *unstructured.Unstructured) { + t.Helper() + for key, want := range map[string]float64{"isl": 1024, "osl": 128, "concurrency": 10} { + if got := v1beta2Number(t, obj, "spec", "workload", "static", key); got != want { + t.Fatalf("workload.static.%s = %v, want %v", key, got, want) + } + } + v1beta2Absent(t, obj, "spec", "workload", "static", "kvLoadRatio") + v1beta2Absent(t, obj, "spec", "objective", "sla") + if got := v1beta2String(t, obj, "spec", "objective", "mode"); got != "Optimize" { + t.Fatalf("objective.mode = %q, want Optimize", got) + } +} + +// expectV1beta2AdmissionDefaults checks what admission supplies from the environment. +func expectV1beta2AdmissionDefaults(t *testing.T, obj *unstructured.Unstructured) { + t.Helper() + if got := v1beta2String(t, obj, "spec", "objective", "metric"); got != "Throughput" { + t.Fatalf("objective.metric = %q, want Throughput", got) + } + if got := v1beta2String(t, obj, "spec", "aisimulate", "image"); got != admissionAISimulateImage { + t.Fatalf("aisimulate.image = %q, want %q", got, admissionAISimulateImage) + } + if got := v1beta2String(t, obj, "spec", "hardware", "gpu", "sku"); got != "h200_sxm" { + t.Fatalf("hardware.gpu.sku = %q, want h200_sxm", got) + } + if got := v1beta2Number(t, obj, "spec", "hardware", "gpu", "budget"); got != 8 { + t.Fatalf("hardware.gpu.budget = %v, want 8", got) + } +} diff --git a/deploy/operator/internal/webhook/validation/planner_runtime_features.go b/deploy/operator/internal/webhook/validation/planner_runtime_features.go index 1e772c501f72..c61ff68ec318 100644 --- a/deploy/operator/internal/webhook/validation/planner_runtime_features.go +++ b/deploy/operator/internal/webhook/validation/planner_runtime_features.go @@ -13,7 +13,7 @@ import ( "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo" runtimefeatures "github.com/ai-dynamo/dynamo/deploy/operator/internal/features/runtime" - "github.com/ai-dynamo/dynamo/deploy/operator/internal/runtimeversion" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/versionoverride" "k8s.io/apimachinery/pkg/util/validation/field" ) @@ -117,7 +117,7 @@ func rolePodTemplatesPlannerRuntimeErrorDetail(planners []plannerRuntimeContract minimum := runtimefeatures.PlannerDGDComponentStatus.MinRuntimeVersion.String() for _, planner := range planners { - version, err := runtimeversion.Resolve(planner.image, planner.runtimeVersionOverride) + version, err := versionoverride.Resolve(planner.image, planner.runtimeVersionOverride) if err != nil { return fmt.Sprintf( "role-specific PodTemplates require every planner component to use Dynamo runtime %s or later; planner component %q has no resolvable runtime version: %v", diff --git a/deploy/operator/internal/webhook/validation/shared_helpers.go b/deploy/operator/internal/webhook/validation/shared_helpers.go index 053c8e1f0041..19e302c7bfac 100644 --- a/deploy/operator/internal/webhook/validation/shared_helpers.go +++ b/deploy/operator/internal/webhook/validation/shared_helpers.go @@ -30,7 +30,7 @@ import ( "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/epp" "github.com/ai-dynamo/dynamo/deploy/operator/internal/features" runtimefeatures "github.com/ai-dynamo/dynamo/deploy/operator/internal/features/runtime" - "github.com/ai-dynamo/dynamo/deploy/operator/internal/runtimeversion" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/versionoverride" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/validation/field" @@ -153,7 +153,7 @@ func runtimeVersionOverrideRequired(image, override string) bool { if override != "" { return false } - _, err := runtimeversion.ParseImageVersion(image) + _, err := versionoverride.ParseImageVersion(image) return err != nil } @@ -196,7 +196,7 @@ func eppRuntimeCompatibilityError(contract eppRuntimeContract, eppConfigPath *fi return nil } - version, err := runtimeversion.Resolve(contract.image, contract.runtimeVersionOverride) + version, err := versionoverride.Resolve(contract.image, contract.runtimeVersionOverride) if err != nil { return nil } diff --git a/deploy/operator/internal/webhook/validation/suite_envtest_test.go b/deploy/operator/internal/webhook/validation/suite_envtest_test.go index 12e4485f2f6b..c09e02c99d5b 100644 --- a/deploy/operator/internal/webhook/validation/suite_envtest_test.go +++ b/deploy/operator/internal/webhook/validation/suite_envtest_test.go @@ -22,6 +22,7 @@ import ( "github.com/ai-dynamo/dynamo/deploy/operator/internal/features" "github.com/ai-dynamo/dynamo/deploy/operator/internal/provideroverride" "github.com/ai-dynamo/dynamo/deploy/operator/internal/testing/operatorenv" + webhookdefaulting "github.com/ai-dynamo/dynamo/deploy/operator/internal/webhook/defaulting" webhooksetup "github.com/ai-dynamo/dynamo/deploy/operator/internal/webhook/setup" grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" @@ -43,6 +44,7 @@ const ( frontendImage150 = "registry.example/dynamo-frontend:1.5.0" legacyEPPImage140 = "registry.example/epp-image:1.4.0" legacySeedUsername = "operatorenv-legacy-seeder" + admissionAISimulateImage = "registry.example/aisimulate:1.2.0" ) func groveProviderOverride(target, value string) *nvidiacomv1beta1.ProviderOverride { @@ -65,6 +67,9 @@ var ( // Fixture barriers observe the same cache used by webhook validation. admissionReader client.Reader + // Hardware that the v1beta2 defaulter discovers; tests set it before each case. + admissionHardware = &fakeHardwareDiscoverer{} + // Admission cases must remain sequential because they share this gate and a cluster-scoped topology fixture. admissionGate = &mutableFeatureGate{} admissionEnv = operatorenv.New(operatorenv.Options{ @@ -91,9 +96,33 @@ func setupAdmissionWebhooks(mgr ctrl.Manager, opts operatorenv.WebhookSetupOptio OperatorVersion: opts.OperatorVersion, OperatorPrincipal: opts.OperatorPrincipal, Gate: admissionGate, + DGDRV1beta2: webhookdefaulting.DGDRV1beta2Options{ + AISimulateImage: admissionAISimulateImage, + Hardware: admissionHardware, + }, }) } +// fakeHardwareDiscoverer stands in for the cluster's GPU discovery. Cases run sequentially, +// so a plain mutex-guarded value is enough. +type fakeHardwareDiscoverer struct { + mu sync.Mutex + found webhookdefaulting.DiscoveredHardware + err error +} + +func (f *fakeHardwareDiscoverer) set(found webhookdefaulting.DiscoveredHardware, err error) { + f.mu.Lock() + defer f.mu.Unlock() + f.found, f.err = found, err +} + +func (f *fakeHardwareDiscoverer) Discover(context.Context, string) (webhookdefaulting.DiscoveredHardware, error) { + f.mu.Lock() + defer f.mu.Unlock() + return f.found, f.err +} + type mutableFeatureGate struct { mu sync.RWMutex gates features.Gate @@ -122,6 +151,10 @@ type admissionTestCase struct { terminating bool username string + // beforeUpdate runs after the old state is seeded and before the update is submitted. Use + // it to change what the webhooks see between the two writes, such as hardware discovery. + beforeUpdate func(*testing.T) + wantSchemaError string wantCELError string wantStatusError string @@ -166,6 +199,10 @@ func runAdmissionTest(t *testing.T, test admissionTestCase) *unstructured.Unstru old = beginTermination(t, seedClient, old) } + if test.beforeUpdate != nil { + test.beforeUpdate(t) + } + t.Log("Submit the update request through the Kubernetes API server") warnings.clear() oldFixture, _ := admissionObject(t, test.oldObject, env.Namespace(), nil) diff --git a/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md b/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md index 6331c345674b..29f2c9448df2 100644 --- a/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md +++ b/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md @@ -3242,8 +3242,8 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | -| `image` _string_ | Image selects a Dynamo-compatible AISimulate runner. When omitted, admission
and execution defaults are resolved outside this API. | | MinLength: 1
Optional: \{\}
| -| `versionOverride` _string_ | VersionOverride declares the AISimulate compatibility version when it
cannot be inferred from the effective image. | | MinLength: 1
Optional: \{\}
| +| `image` _string_ | Image selects a Dynamo-compatible AISimulate runner. When omitted, admission
persists the operator's configured image. | | MinLength: 1
Optional: \{\}
| +| `versionOverride` _string_ | VersionOverride declares the AISimulate compatibility version as three numeric parts
(for example 0.13.0), not a semver version: no v prefix, pre-release or build suffix. It is required
when the image tag is not a version and wins over a version in the tag. | | MinLength: 1
Pattern: `^(0\|[1-9][0-9]\{0,3\})\.(0\|[1-9][0-9]\{0,3\})\.(0\|[1-9][0-9]\{0,3\})$`
Optional: \{\}
| | `parameters` _[RawExtension](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#rawextension-runtime-pkg)_ | Parameters contains native AISimulate input. Field names and nesting use
the upstream AISimulate JSON contract, including snake_case names. | | Type: object
Optional: \{\}
| @@ -3438,8 +3438,8 @@ _Appears in:_ | `model` _[ModelSpec](#modelspec)_ | Model identifies the model whose deployment configurations are evaluated. | | Optional: \{\}
| | `backends` _[Backend](#backend) array_ | Backends lists one or more inference backends searched by AISimulate and used by generated candidates. | | Enum: [vllm sglang trtllm]
MinItems: 1
Optional: \{\}
| | `hardware` _[HardwareSpec](#v1beta2-hardwarespec)_ | Hardware bounds the accelerator configurations evaluated by the search.
Admission discovers any omitted hardware values from the cluster. | | Optional: \{\}
| -| `workload` _[WorkloadSpec](#v1beta2-workloadspec)_ | Workload defines the synthetic traffic model used for every candidate evaluation. | | Optional: \{\}
| -| `objective` _[ObjectiveSpec](#objectivespec)_ | Objective defines scalar optimization or a multi-objective Pareto search.
When omitted, the schema supplies the default Optimize mode and SLA;
admission supplies the conditional scalar metric. | | Optional: \{\}
| +| `workload` _[WorkloadSpec](#v1beta2-workloadspec)_ | Workload defines the synthetic traffic model used for every candidate evaluation.
When omitted, admission supplies a static shape of ISL 1024, OSL 128 and concurrency 10;
an explicit empty workload receives the same shape from the schema. An explicit
workload.static: \{\} opts out of that shape, leaving the workload to
spec.aisimulate.parameters.workload. spec.aisimulate.parameters.workload overrides the
fields it names. | | Optional: \{\}
| +| `objective` _[ObjectiveSpec](#objectivespec)_ | Objective defines scalar optimization or a multi-objective Pareto search.
When omitted, admission supplies the object. Its mode defaults to Optimize, and admission
supplies the conditional scalar metric. | | Optional: \{\}
| | `aisimulate` _[AISimulateSpec](#aisimulatespec)_ | AISimulate selects the execution artifact and supplies native AISimulate input. | | Optional: \{\}
| | `recommendation` _[RecommendationSpec](#recommendationspec)_ | Recommendation controls bounded projection into DGDC resources. | | Optional: \{\}
| @@ -3480,8 +3480,8 @@ _Appears in:_ | `model` _[ModelSpec](#modelspec)_ | Model identifies the model whose deployment configurations are evaluated. | | Optional: \{\}
| | `backends` _[Backend](#backend) array_ | Backends lists one or more inference backends searched by AISimulate and used by generated candidates. | | Enum: [vllm sglang trtllm]
MinItems: 1
Optional: \{\}
| | `hardware` _[HardwareSpec](#v1beta2-hardwarespec)_ | Hardware bounds the accelerator configurations evaluated by the search.
Admission discovers any omitted hardware values from the cluster. | | Optional: \{\}
| -| `workload` _[WorkloadSpec](#v1beta2-workloadspec)_ | Workload defines the synthetic traffic model used for every candidate evaluation. | | Optional: \{\}
| -| `objective` _[ObjectiveSpec](#objectivespec)_ | Objective defines scalar optimization or a multi-objective Pareto search.
When omitted, the schema supplies the default Optimize mode and SLA;
admission supplies the conditional scalar metric. | | Optional: \{\}
| +| `workload` _[WorkloadSpec](#v1beta2-workloadspec)_ | Workload defines the synthetic traffic model used for every candidate evaluation.
When omitted, admission supplies a static shape of ISL 1024, OSL 128 and concurrency 10;
an explicit empty workload receives the same shape from the schema. An explicit
workload.static: \{\} opts out of that shape, leaving the workload to
spec.aisimulate.parameters.workload. spec.aisimulate.parameters.workload overrides the
fields it names. | | Optional: \{\}
| +| `objective` _[ObjectiveSpec](#objectivespec)_ | Objective defines scalar optimization or a multi-objective Pareto search.
When omitted, admission supplies the object. Its mode defaults to Optimize, and admission
supplies the conditional scalar metric. | | Optional: \{\}
| | `aisimulate` _[AISimulateSpec](#aisimulatespec)_ | AISimulate selects the execution artifact and supplies native AISimulate input. | | Optional: \{\}
| | `recommendation` _[RecommendationSpec](#recommendationspec)_ | Recommendation controls bounded projection into DGDC resources. | | Optional: \{\}
| | `v1beta1` _[JSON](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#json-v1-apiextensions-k8s-io)_ | V1Beta1 losslessly preserves a native v1beta1 spec while this object is
represented through v1beta2. It is mutually exclusive with native fields.
This field is a read-only conversion view; modify the request through the
native v1beta1 API. | | Type: object
Optional: \{\}
| @@ -3543,8 +3543,8 @@ _Appears in:_ | `model` _[ModelSpec](#modelspec)_ | Model identifies the model whose deployment configurations are evaluated. | | Optional: \{\}
| | `backends` _[Backend](#backend) array_ | Backends lists one or more inference backends searched by AISimulate and used by generated candidates. | | Enum: [vllm sglang trtllm]
MinItems: 1
Optional: \{\}
| | `hardware` _[HardwareSpec](#v1beta2-hardwarespec)_ | Hardware bounds the accelerator configurations evaluated by the search.
Admission discovers any omitted hardware values from the cluster. | | Optional: \{\}
| -| `workload` _[WorkloadSpec](#v1beta2-workloadspec)_ | Workload defines the synthetic traffic model used for every candidate evaluation. | | Optional: \{\}
| -| `objective` _[ObjectiveSpec](#objectivespec)_ | Objective defines scalar optimization or a multi-objective Pareto search.
When omitted, the schema supplies the default Optimize mode and SLA;
admission supplies the conditional scalar metric. | | Optional: \{\}
| +| `workload` _[WorkloadSpec](#v1beta2-workloadspec)_ | Workload defines the synthetic traffic model used for every candidate evaluation.
When omitted, admission supplies a static shape of ISL 1024, OSL 128 and concurrency 10;
an explicit empty workload receives the same shape from the schema. An explicit
workload.static: \{\} opts out of that shape, leaving the workload to
spec.aisimulate.parameters.workload. spec.aisimulate.parameters.workload overrides the
fields it names. | | Optional: \{\}
| +| `objective` _[ObjectiveSpec](#objectivespec)_ | Objective defines scalar optimization or a multi-objective Pareto search.
When omitted, admission supplies the object. Its mode defaults to Optimize, and admission
supplies the conditional scalar metric. | | Optional: \{\}
| | `aisimulate` _[AISimulateSpec](#aisimulatespec)_ | AISimulate selects the execution artifact and supplies native AISimulate input. | | Optional: \{\}
| | `recommendation` _[RecommendationSpec](#recommendationspec)_ | Recommendation controls bounded projection into DGDC resources. | | Optional: \{\}
| @@ -3686,8 +3686,8 @@ _Appears in:_ | `model` _[ModelSpec](#modelspec)_ | Model identifies the model whose deployment configurations are evaluated. | | Optional: \{\}
| | `backends` _[Backend](#backend) array_ | Backends lists one or more inference backends searched by AISimulate and used by generated candidates. | | Enum: [vllm sglang trtllm]
MinItems: 1
Optional: \{\}
| | `hardware` _[HardwareSpec](#v1beta2-hardwarespec)_ | Hardware bounds the accelerator configurations evaluated by the search.
Admission discovers any omitted hardware values from the cluster. | | Optional: \{\}
| -| `workload` _[WorkloadSpec](#v1beta2-workloadspec)_ | Workload defines the synthetic traffic model used for every candidate evaluation. | | Optional: \{\}
| -| `objective` _[ObjectiveSpec](#objectivespec)_ | Objective defines scalar optimization or a multi-objective Pareto search.
When omitted, the schema supplies the default Optimize mode and SLA;
admission supplies the conditional scalar metric. | | Optional: \{\}
| +| `workload` _[WorkloadSpec](#v1beta2-workloadspec)_ | Workload defines the synthetic traffic model used for every candidate evaluation.
When omitted, admission supplies a static shape of ISL 1024, OSL 128 and concurrency 10;
an explicit empty workload receives the same shape from the schema. An explicit
workload.static: \{\} opts out of that shape, leaving the workload to
spec.aisimulate.parameters.workload. spec.aisimulate.parameters.workload overrides the
fields it names. | | Optional: \{\}
| +| `objective` _[ObjectiveSpec](#objectivespec)_ | Objective defines scalar optimization or a multi-objective Pareto search.
When omitted, admission supplies the object. Its mode defaults to Optimize, and admission
supplies the conditional scalar metric. | | Optional: \{\}
| | `aisimulate` _[AISimulateSpec](#aisimulatespec)_ | AISimulate selects the execution artifact and supplies native AISimulate input. | | Optional: \{\}
| | `recommendation` _[RecommendationSpec](#recommendationspec)_ | Recommendation controls bounded projection into DGDC resources. | | Optional: \{\}
| @@ -3762,7 +3762,7 @@ _Appears in:_ | `mode` _[ObjectiveMode](#objectivemode)_ | Mode selects scalar optimization or a Pareto search. | Optimize | Enum: [Optimize Pareto]
Optional: \{\}
| | `metric` _[ObjectiveMetric](#objectivemetric)_ | Metric is the scalar target when mode is Optimize.
When omitted in Optimize mode, admission defaults it to ThroughputPerGPU. | | Enum: [Throughput ThroughputPerGPU ThroughputPerUser TTFT E2ELatency Goodput GoodputPerGPU]
Optional: \{\}
| | `metrics` _[ObjectiveMetric](#objectivemetric) array_ | Metrics lists two or more distinct scalar targets when mode is Pareto. | | Enum: [Throughput ThroughputPerGPU ThroughputPerUser TTFT E2ELatency Goodput GoodputPerGPU]
MaxItems: 7
MinItems: 2
Optional: \{\}
| -| `sla` _[ObjectiveSLASpec](#objectiveslaspec)_ | SLA defines per-request latency bounds used to calculate goodput. | \{ itlMs:30 ttftMs:2000 \} | Optional: \{\}
| +| `sla` _[ObjectiveSLASpec](#objectiveslaspec)_ | SLA defines per-request latency bounds used to calculate goodput. When omitted, no
bounds apply. spec.aisimulate.parameters.goal.sla overrides the bounds it names. | | Optional: \{\}
| #### RecommendationSpec @@ -3855,7 +3855,7 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | -| `static` _[StaticWorkloadSpec](#staticworkloadspec)_ | Static defines one of AISimulate's synthetic workload shapes. When omitted,
it searches the full KV-load ratio range at ISL 4000 and OSL 1000.
An explicit empty object opts out of this underlay. | \{ isl:4000 kvLoadRatio:[0 1] osl:1000 \} | Optional: \{\}
| +| `static` _[StaticWorkloadSpec](#staticworkloadspec)_ | Static defines one of AISimulate's synthetic workload shapes. Its fields are not
defaulted individually: the default belongs to spec.workload as a whole, so an
explicit workload object keeps exactly the fields it names. | \{ concurrency:10 isl:1024 osl:128 \} | Optional: \{\}
| diff --git a/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx b/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx index 571fd8ba74d1..dc1496b529b4 100644 --- a/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx +++ b/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx @@ -4531,13 +4531,13 @@ AISimulateSpec selects the AISimulate execution artifact and supplies native inp **Appears in:** [DynamoGraphDeploymentRequestNativeSpec](#v1beta2-dynamographdeploymentrequestnativespec), [DynamoGraphDeploymentRequestSpec](#v1beta2-dynamographdeploymentrequestspec), [DynamoGraphDeploymentRunSpec](#dynamographdeploymentrunspec), [NativeSearchInput](#nativesearchinput) -Image selects a Dynamo-compatible AISimulate runner. When omitted, admission and execution defaults are resolved outside this API. +Image selects a Dynamo-compatible AISimulate runner. When omitted, admission persists the operator's configured image. **Validation:** MinLength: 1 Optional: \{\} -VersionOverride declares the AISimulate compatibility version when it cannot be inferred from the effective image. -**Validation:** MinLength: 1 Optional: \{\} +VersionOverride declares the AISimulate compatibility version as three numeric parts (for example 0.13.0), not a semver version: no v prefix, pre-release or build suffix. It is required when the image tag is not a version and wins over a version in the tag. +**Validation:** MinLength: 1 Pattern: `^(0|[1-9][0-9]\{0,3\})\.(0|[1-9][0-9]\{0,3\})\.(0|[1-9][0-9]\{0,3\})$` Optional: \{\} @@ -4806,13 +4806,13 @@ See [HardwareSpec](#v1beta2-hardwarespec). -Workload defines the synthetic traffic model used for every candidate evaluation. +Workload defines the synthetic traffic model used for every candidate evaluation. When omitted, admission supplies a static shape of ISL 1024, OSL 128 and concurrency 10; an explicit empty workload receives the same shape from the schema. An explicit workload.static: \{\} opts out of that shape, leaving the workload to spec.aisimulate.parameters.workload. spec.aisimulate.parameters.workload overrides the fields it names. See [WorkloadSpec](#v1beta2-workloadspec). **Validation:** Optional: \{\} -Objective defines scalar optimization or a multi-objective Pareto search. When omitted, the schema supplies the default Optimize mode and SLA; admission supplies the conditional scalar metric. +Objective defines scalar optimization or a multi-objective Pareto search. When omitted, admission supplies the object. Its mode defaults to Optimize, and admission supplies the conditional scalar metric. See [ObjectiveSpec](#objectivespec). **Validation:** Optional: \{\} @@ -4888,13 +4888,13 @@ See [HardwareSpec](#v1beta2-hardwarespec). -Workload defines the synthetic traffic model used for every candidate evaluation. +Workload defines the synthetic traffic model used for every candidate evaluation. When omitted, admission supplies a static shape of ISL 1024, OSL 128 and concurrency 10; an explicit empty workload receives the same shape from the schema. An explicit workload.static: \{\} opts out of that shape, leaving the workload to spec.aisimulate.parameters.workload. spec.aisimulate.parameters.workload overrides the fields it names. See [WorkloadSpec](#v1beta2-workloadspec). **Validation:** Optional: \{\} -Objective defines scalar optimization or a multi-objective Pareto search. When omitted, the schema supplies the default Optimize mode and SLA; admission supplies the conditional scalar metric. +Objective defines scalar optimization or a multi-objective Pareto search. When omitted, admission supplies the object. Its mode defaults to Optimize, and admission supplies the conditional scalar metric. See [ObjectiveSpec](#objectivespec). **Validation:** Optional: \{\} @@ -5010,13 +5010,13 @@ See [HardwareSpec](#v1beta2-hardwarespec). -Workload defines the synthetic traffic model used for every candidate evaluation. +Workload defines the synthetic traffic model used for every candidate evaluation. When omitted, admission supplies a static shape of ISL 1024, OSL 128 and concurrency 10; an explicit empty workload receives the same shape from the schema. An explicit workload.static: \{\} opts out of that shape, leaving the workload to spec.aisimulate.parameters.workload. spec.aisimulate.parameters.workload overrides the fields it names. See [WorkloadSpec](#v1beta2-workloadspec). **Validation:** Optional: \{\} -Objective defines scalar optimization or a multi-objective Pareto search. When omitted, the schema supplies the default Optimize mode and SLA; admission supplies the conditional scalar metric. +Objective defines scalar optimization or a multi-objective Pareto search. When omitted, admission supplies the object. Its mode defaults to Optimize, and admission supplies the conditional scalar metric. See [ObjectiveSpec](#objectivespec). **Validation:** Optional: \{\} @@ -5209,13 +5209,13 @@ See [HardwareSpec](#v1beta2-hardwarespec). -Workload defines the synthetic traffic model used for every candidate evaluation. +Workload defines the synthetic traffic model used for every candidate evaluation. When omitted, admission supplies a static shape of ISL 1024, OSL 128 and concurrency 10; an explicit empty workload receives the same shape from the schema. An explicit workload.static: \{\} opts out of that shape, leaving the workload to spec.aisimulate.parameters.workload. spec.aisimulate.parameters.workload overrides the fields it names. See [WorkloadSpec](#v1beta2-workloadspec). **Validation:** Optional: \{\} -Objective defines scalar optimization or a multi-objective Pareto search. When omitted, the schema supplies the default Optimize mode and SLA; admission supplies the conditional scalar metric. +Objective defines scalar optimization or a multi-objective Pareto search. When omitted, admission supplies the object. Its mode defaults to Optimize, and admission supplies the conditional scalar metric. See [ObjectiveSpec](#objectivespec). **Validation:** Optional: \{\} @@ -5307,8 +5307,8 @@ See [ObjectiveMetric](#objectivemetric). **Validation:** Enum: [Throughput ThroughputPerGPU ThroughputPerUser TTFT E2ELatency Goodput GoodputPerGPU] MaxItems: 7 MinItems: 2 Optional: \{\} - -SLA defines per-request latency bounds used to calculate goodput. + +SLA defines per-request latency bounds used to calculate goodput. When omitted, no bounds apply. spec.aisimulate.parameters.goal.sla overrides the bounds it names. See [ObjectiveSLASpec](#objectiveslaspec). **Validation:** Optional: \{\} @@ -5407,8 +5407,8 @@ WorkloadSpec defines the synthetic traffic model used for every candidate evalua **Kind:** `type` **Appears in:** [DynamoGraphDeploymentRequestNativeSpec](#v1beta2-dynamographdeploymentrequestnativespec), [DynamoGraphDeploymentRequestSpec](#v1beta2-dynamographdeploymentrequestspec), [DynamoGraphDeploymentRunSpec](#dynamographdeploymentrunspec), [NativeSearchInput](#nativesearchinput) - -Static defines one of AISimulate's synthetic workload shapes. When omitted, it searches the full KV-load ratio range at ISL 4000 and OSL 1000. An explicit empty object opts out of this underlay. + +Static defines one of AISimulate's synthetic workload shapes. Its fields are not defaulted individually: the default belongs to spec.workload as a whole, so an explicit workload object keeps exactly the fields it names. See [StaticWorkloadSpec](#staticworkloadspec). **Validation:** Optional: \{\} From 3457f9f761daa9a2add766174be1592feb0fd3a1 Mon Sep 17 00:00:00 2001 From: devivasudevan <49675305+devivasudevan@users.noreply.github.com> Date: Wed, 21 Oct 2026 21:47:33 +0200 Subject: [PATCH 19/40] docs(operator): fix stale v1beta2 comments; test a full update of an unchanged carrier Objective.metric: the Optimize default is Throughput, not ThroughputPerGPU. Workload: the fixed-concurrency default creates neither sweep.max_trials nor a KV-load ratio search. The carrier test now also writes the unchanged foreign view back with a full update and requires the spec to be byte-stable and the generation unchanged, in both directions. Signed-off-by: devivasudevan <49675305+devivasudevan@users.noreply.github.com> --- .../dynamographdeploymentrequest_types.go | 4 ++- ...dia.com_dynamographdeploymentrequests.yaml | 4 ++- .../nvidia.com_dynamographdeploymentruns.yaml | 4 ++- ...hdeploymentrequest_carrier_envtest_test.go | 26 +++++++++++++++++++ .../additional-resources/api-reference-k8s.md | 10 +++---- .../kubernetes-api/full-api-reference.mdx | 10 +++---- 6 files changed, 45 insertions(+), 13 deletions(-) diff --git a/deploy/operator/api/v1beta2/dynamographdeploymentrequest_types.go b/deploy/operator/api/v1beta2/dynamographdeploymentrequest_types.go index 2a6e007ac3d4..703825595547 100644 --- a/deploy/operator/api/v1beta2/dynamographdeploymentrequest_types.go +++ b/deploy/operator/api/v1beta2/dynamographdeploymentrequest_types.go @@ -165,7 +165,7 @@ type ObjectiveSpec struct { Mode ObjectiveMode `json:"mode,omitempty"` // Metric is the scalar target when mode is Optimize. - // When omitted in Optimize mode, admission defaults it to ThroughputPerGPU. + // When omitted in Optimize mode, admission defaults it to Throughput. // +optional Metric *ObjectiveMetric `json:"metric,omitempty"` @@ -241,6 +241,8 @@ type NativeSearchInput struct { // workload.static: {} opts out of that shape, leaving the workload to // spec.aisimulate.parameters.workload. spec.aisimulate.parameters.workload overrides the // fields it names. + // The default evaluates a fixed concurrency, so it creates neither sweep.max_trials nor a + // KV-load ratio search. // +optional Workload *WorkloadSpec `json:"workload,omitempty"` diff --git a/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentrequests.yaml b/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentrequests.yaml index 0c4c79c78372..65b19e735a05 100644 --- a/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentrequests.yaml +++ b/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentrequests.yaml @@ -9667,7 +9667,7 @@ spec: metric: description: |- Metric is the scalar target when mode is Optimize. - When omitted in Optimize mode, admission defaults it to ThroughputPerGPU. + When omitted in Optimize mode, admission defaults it to Throughput. enum: - Throughput - ThroughputPerGPU @@ -9763,6 +9763,8 @@ spec: workload.static: {} opts out of that shape, leaving the workload to spec.aisimulate.parameters.workload. spec.aisimulate.parameters.workload overrides the fields it names. + The default evaluates a fixed concurrency, so it creates neither sweep.max_trials nor a + KV-load ratio search. properties: static: default: diff --git a/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentruns.yaml b/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentruns.yaml index eb90a63fc2ac..7d7cd9bae305 100644 --- a/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentruns.yaml +++ b/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentruns.yaml @@ -139,7 +139,7 @@ spec: metric: description: |- Metric is the scalar target when mode is Optimize. - When omitted in Optimize mode, admission defaults it to ThroughputPerGPU. + When omitted in Optimize mode, admission defaults it to Throughput. enum: - Throughput - ThroughputPerGPU @@ -227,6 +227,8 @@ spec: workload.static: {} opts out of that shape, leaving the workload to spec.aisimulate.parameters.workload. spec.aisimulate.parameters.workload overrides the fields it names. + The default evaluates a fixed concurrency, so it creates neither sweep.max_trials nor a + KV-load ratio search. properties: static: default: diff --git a/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_carrier_envtest_test.go b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_carrier_envtest_test.go index 2d44cc346ab5..843b415057aa 100644 --- a/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_carrier_envtest_test.go +++ b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_carrier_envtest_test.go @@ -1,6 +1,8 @@ package validation_test import ( + "bytes" + "encoding/json" "sort" "testing" @@ -72,4 +74,28 @@ func carrierRoundTrip(t *testing.T, ownerFixture, otherFixture runtime.Object, c if after.GetGeneration() != created.GetGeneration() { t.Fatalf("generation = %d, want %d: a metadata-only patch rewrote the spec", after.GetGeneration(), created.GetGeneration()) } + + t.Log("Write the unchanged foreign view back with a full update; the spec must be byte-stable") + beforeSpec, err := json.Marshal(foreign.Object["spec"]) + if err != nil { + t.Fatalf("encode the foreign spec: %v", err) + } + current, err := otherClient.Get(t.Context(), name, metav1.GetOptions{}) + if err != nil { + t.Fatalf("read the foreign view before the update: %v", err) + } + updated, err := otherClient.Update(t.Context(), current, metav1.UpdateOptions{}) + if err != nil { + t.Fatalf("full update of the unchanged foreign view: %v", err) + } + afterSpec, err := json.Marshal(updated.Object["spec"]) + if err != nil { + t.Fatalf("encode the updated spec: %v", err) + } + if !bytes.Equal(beforeSpec, afterSpec) { + t.Fatalf("the foreign spec changed across an unchanged update:\nbefore: %s\nafter: %s", beforeSpec, afterSpec) + } + if updated.GetGeneration() != created.GetGeneration() { + t.Fatalf("generation = %d, want %d: an unchanged update rewrote the spec", updated.GetGeneration(), created.GetGeneration()) + } } diff --git a/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md b/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md index 29f2c9448df2..2fb7313d7700 100644 --- a/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md +++ b/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md @@ -3438,7 +3438,7 @@ _Appears in:_ | `model` _[ModelSpec](#modelspec)_ | Model identifies the model whose deployment configurations are evaluated. | | Optional: \{\}
| | `backends` _[Backend](#backend) array_ | Backends lists one or more inference backends searched by AISimulate and used by generated candidates. | | Enum: [vllm sglang trtllm]
MinItems: 1
Optional: \{\}
| | `hardware` _[HardwareSpec](#v1beta2-hardwarespec)_ | Hardware bounds the accelerator configurations evaluated by the search.
Admission discovers any omitted hardware values from the cluster. | | Optional: \{\}
| -| `workload` _[WorkloadSpec](#v1beta2-workloadspec)_ | Workload defines the synthetic traffic model used for every candidate evaluation.
When omitted, admission supplies a static shape of ISL 1024, OSL 128 and concurrency 10;
an explicit empty workload receives the same shape from the schema. An explicit
workload.static: \{\} opts out of that shape, leaving the workload to
spec.aisimulate.parameters.workload. spec.aisimulate.parameters.workload overrides the
fields it names. | | Optional: \{\}
| +| `workload` _[WorkloadSpec](#v1beta2-workloadspec)_ | Workload defines the synthetic traffic model used for every candidate evaluation.
When omitted, admission supplies a static shape of ISL 1024, OSL 128 and concurrency 10;
an explicit empty workload receives the same shape from the schema. An explicit
workload.static: \{\} opts out of that shape, leaving the workload to
spec.aisimulate.parameters.workload. spec.aisimulate.parameters.workload overrides the
fields it names.
The default evaluates a fixed concurrency, so it creates neither sweep.max_trials nor a
KV-load ratio search. | | Optional: \{\}
| | `objective` _[ObjectiveSpec](#objectivespec)_ | Objective defines scalar optimization or a multi-objective Pareto search.
When omitted, admission supplies the object. Its mode defaults to Optimize, and admission
supplies the conditional scalar metric. | | Optional: \{\}
| | `aisimulate` _[AISimulateSpec](#aisimulatespec)_ | AISimulate selects the execution artifact and supplies native AISimulate input. | | Optional: \{\}
| | `recommendation` _[RecommendationSpec](#recommendationspec)_ | Recommendation controls bounded projection into DGDC resources. | | Optional: \{\}
| @@ -3480,7 +3480,7 @@ _Appears in:_ | `model` _[ModelSpec](#modelspec)_ | Model identifies the model whose deployment configurations are evaluated. | | Optional: \{\}
| | `backends` _[Backend](#backend) array_ | Backends lists one or more inference backends searched by AISimulate and used by generated candidates. | | Enum: [vllm sglang trtllm]
MinItems: 1
Optional: \{\}
| | `hardware` _[HardwareSpec](#v1beta2-hardwarespec)_ | Hardware bounds the accelerator configurations evaluated by the search.
Admission discovers any omitted hardware values from the cluster. | | Optional: \{\}
| -| `workload` _[WorkloadSpec](#v1beta2-workloadspec)_ | Workload defines the synthetic traffic model used for every candidate evaluation.
When omitted, admission supplies a static shape of ISL 1024, OSL 128 and concurrency 10;
an explicit empty workload receives the same shape from the schema. An explicit
workload.static: \{\} opts out of that shape, leaving the workload to
spec.aisimulate.parameters.workload. spec.aisimulate.parameters.workload overrides the
fields it names. | | Optional: \{\}
| +| `workload` _[WorkloadSpec](#v1beta2-workloadspec)_ | Workload defines the synthetic traffic model used for every candidate evaluation.
When omitted, admission supplies a static shape of ISL 1024, OSL 128 and concurrency 10;
an explicit empty workload receives the same shape from the schema. An explicit
workload.static: \{\} opts out of that shape, leaving the workload to
spec.aisimulate.parameters.workload. spec.aisimulate.parameters.workload overrides the
fields it names.
The default evaluates a fixed concurrency, so it creates neither sweep.max_trials nor a
KV-load ratio search. | | Optional: \{\}
| | `objective` _[ObjectiveSpec](#objectivespec)_ | Objective defines scalar optimization or a multi-objective Pareto search.
When omitted, admission supplies the object. Its mode defaults to Optimize, and admission
supplies the conditional scalar metric. | | Optional: \{\}
| | `aisimulate` _[AISimulateSpec](#aisimulatespec)_ | AISimulate selects the execution artifact and supplies native AISimulate input. | | Optional: \{\}
| | `recommendation` _[RecommendationSpec](#recommendationspec)_ | Recommendation controls bounded projection into DGDC resources. | | Optional: \{\}
| @@ -3543,7 +3543,7 @@ _Appears in:_ | `model` _[ModelSpec](#modelspec)_ | Model identifies the model whose deployment configurations are evaluated. | | Optional: \{\}
| | `backends` _[Backend](#backend) array_ | Backends lists one or more inference backends searched by AISimulate and used by generated candidates. | | Enum: [vllm sglang trtllm]
MinItems: 1
Optional: \{\}
| | `hardware` _[HardwareSpec](#v1beta2-hardwarespec)_ | Hardware bounds the accelerator configurations evaluated by the search.
Admission discovers any omitted hardware values from the cluster. | | Optional: \{\}
| -| `workload` _[WorkloadSpec](#v1beta2-workloadspec)_ | Workload defines the synthetic traffic model used for every candidate evaluation.
When omitted, admission supplies a static shape of ISL 1024, OSL 128 and concurrency 10;
an explicit empty workload receives the same shape from the schema. An explicit
workload.static: \{\} opts out of that shape, leaving the workload to
spec.aisimulate.parameters.workload. spec.aisimulate.parameters.workload overrides the
fields it names. | | Optional: \{\}
| +| `workload` _[WorkloadSpec](#v1beta2-workloadspec)_ | Workload defines the synthetic traffic model used for every candidate evaluation.
When omitted, admission supplies a static shape of ISL 1024, OSL 128 and concurrency 10;
an explicit empty workload receives the same shape from the schema. An explicit
workload.static: \{\} opts out of that shape, leaving the workload to
spec.aisimulate.parameters.workload. spec.aisimulate.parameters.workload overrides the
fields it names.
The default evaluates a fixed concurrency, so it creates neither sweep.max_trials nor a
KV-load ratio search. | | Optional: \{\}
| | `objective` _[ObjectiveSpec](#objectivespec)_ | Objective defines scalar optimization or a multi-objective Pareto search.
When omitted, admission supplies the object. Its mode defaults to Optimize, and admission
supplies the conditional scalar metric. | | Optional: \{\}
| | `aisimulate` _[AISimulateSpec](#aisimulatespec)_ | AISimulate selects the execution artifact and supplies native AISimulate input. | | Optional: \{\}
| | `recommendation` _[RecommendationSpec](#recommendationspec)_ | Recommendation controls bounded projection into DGDC resources. | | Optional: \{\}
| @@ -3686,7 +3686,7 @@ _Appears in:_ | `model` _[ModelSpec](#modelspec)_ | Model identifies the model whose deployment configurations are evaluated. | | Optional: \{\}
| | `backends` _[Backend](#backend) array_ | Backends lists one or more inference backends searched by AISimulate and used by generated candidates. | | Enum: [vllm sglang trtllm]
MinItems: 1
Optional: \{\}
| | `hardware` _[HardwareSpec](#v1beta2-hardwarespec)_ | Hardware bounds the accelerator configurations evaluated by the search.
Admission discovers any omitted hardware values from the cluster. | | Optional: \{\}
| -| `workload` _[WorkloadSpec](#v1beta2-workloadspec)_ | Workload defines the synthetic traffic model used for every candidate evaluation.
When omitted, admission supplies a static shape of ISL 1024, OSL 128 and concurrency 10;
an explicit empty workload receives the same shape from the schema. An explicit
workload.static: \{\} opts out of that shape, leaving the workload to
spec.aisimulate.parameters.workload. spec.aisimulate.parameters.workload overrides the
fields it names. | | Optional: \{\}
| +| `workload` _[WorkloadSpec](#v1beta2-workloadspec)_ | Workload defines the synthetic traffic model used for every candidate evaluation.
When omitted, admission supplies a static shape of ISL 1024, OSL 128 and concurrency 10;
an explicit empty workload receives the same shape from the schema. An explicit
workload.static: \{\} opts out of that shape, leaving the workload to
spec.aisimulate.parameters.workload. spec.aisimulate.parameters.workload overrides the
fields it names.
The default evaluates a fixed concurrency, so it creates neither sweep.max_trials nor a
KV-load ratio search. | | Optional: \{\}
| | `objective` _[ObjectiveSpec](#objectivespec)_ | Objective defines scalar optimization or a multi-objective Pareto search.
When omitted, admission supplies the object. Its mode defaults to Optimize, and admission
supplies the conditional scalar metric. | | Optional: \{\}
| | `aisimulate` _[AISimulateSpec](#aisimulatespec)_ | AISimulate selects the execution artifact and supplies native AISimulate input. | | Optional: \{\}
| | `recommendation` _[RecommendationSpec](#recommendationspec)_ | Recommendation controls bounded projection into DGDC resources. | | Optional: \{\}
| @@ -3760,7 +3760,7 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | | `mode` _[ObjectiveMode](#objectivemode)_ | Mode selects scalar optimization or a Pareto search. | Optimize | Enum: [Optimize Pareto]
Optional: \{\}
| -| `metric` _[ObjectiveMetric](#objectivemetric)_ | Metric is the scalar target when mode is Optimize.
When omitted in Optimize mode, admission defaults it to ThroughputPerGPU. | | Enum: [Throughput ThroughputPerGPU ThroughputPerUser TTFT E2ELatency Goodput GoodputPerGPU]
Optional: \{\}
| +| `metric` _[ObjectiveMetric](#objectivemetric)_ | Metric is the scalar target when mode is Optimize.
When omitted in Optimize mode, admission defaults it to Throughput. | | Enum: [Throughput ThroughputPerGPU ThroughputPerUser TTFT E2ELatency Goodput GoodputPerGPU]
Optional: \{\}
| | `metrics` _[ObjectiveMetric](#objectivemetric) array_ | Metrics lists two or more distinct scalar targets when mode is Pareto. | | Enum: [Throughput ThroughputPerGPU ThroughputPerUser TTFT E2ELatency Goodput GoodputPerGPU]
MaxItems: 7
MinItems: 2
Optional: \{\}
| | `sla` _[ObjectiveSLASpec](#objectiveslaspec)_ | SLA defines per-request latency bounds used to calculate goodput. When omitted, no
bounds apply. spec.aisimulate.parameters.goal.sla overrides the bounds it names. | | Optional: \{\}
| diff --git a/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx b/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx index dc1496b529b4..dfad335c1aff 100644 --- a/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx +++ b/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx @@ -4806,7 +4806,7 @@ See [HardwareSpec](#v1beta2-hardwarespec). -Workload defines the synthetic traffic model used for every candidate evaluation. When omitted, admission supplies a static shape of ISL 1024, OSL 128 and concurrency 10; an explicit empty workload receives the same shape from the schema. An explicit workload.static: \{\} opts out of that shape, leaving the workload to spec.aisimulate.parameters.workload. spec.aisimulate.parameters.workload overrides the fields it names. +Workload defines the synthetic traffic model used for every candidate evaluation. When omitted, admission supplies a static shape of ISL 1024, OSL 128 and concurrency 10; an explicit empty workload receives the same shape from the schema. An explicit workload.static: \{\} opts out of that shape, leaving the workload to spec.aisimulate.parameters.workload. spec.aisimulate.parameters.workload overrides the fields it names. The default evaluates a fixed concurrency, so it creates neither sweep.max_trials nor a KV-load ratio search. See [WorkloadSpec](#v1beta2-workloadspec). **Validation:** Optional: \{\} @@ -4888,7 +4888,7 @@ See [HardwareSpec](#v1beta2-hardwarespec). -Workload defines the synthetic traffic model used for every candidate evaluation. When omitted, admission supplies a static shape of ISL 1024, OSL 128 and concurrency 10; an explicit empty workload receives the same shape from the schema. An explicit workload.static: \{\} opts out of that shape, leaving the workload to spec.aisimulate.parameters.workload. spec.aisimulate.parameters.workload overrides the fields it names. +Workload defines the synthetic traffic model used for every candidate evaluation. When omitted, admission supplies a static shape of ISL 1024, OSL 128 and concurrency 10; an explicit empty workload receives the same shape from the schema. An explicit workload.static: \{\} opts out of that shape, leaving the workload to spec.aisimulate.parameters.workload. spec.aisimulate.parameters.workload overrides the fields it names. The default evaluates a fixed concurrency, so it creates neither sweep.max_trials nor a KV-load ratio search. See [WorkloadSpec](#v1beta2-workloadspec). **Validation:** Optional: \{\} @@ -5010,7 +5010,7 @@ See [HardwareSpec](#v1beta2-hardwarespec). -Workload defines the synthetic traffic model used for every candidate evaluation. When omitted, admission supplies a static shape of ISL 1024, OSL 128 and concurrency 10; an explicit empty workload receives the same shape from the schema. An explicit workload.static: \{\} opts out of that shape, leaving the workload to spec.aisimulate.parameters.workload. spec.aisimulate.parameters.workload overrides the fields it names. +Workload defines the synthetic traffic model used for every candidate evaluation. When omitted, admission supplies a static shape of ISL 1024, OSL 128 and concurrency 10; an explicit empty workload receives the same shape from the schema. An explicit workload.static: \{\} opts out of that shape, leaving the workload to spec.aisimulate.parameters.workload. spec.aisimulate.parameters.workload overrides the fields it names. The default evaluates a fixed concurrency, so it creates neither sweep.max_trials nor a KV-load ratio search. See [WorkloadSpec](#v1beta2-workloadspec). **Validation:** Optional: \{\} @@ -5209,7 +5209,7 @@ See [HardwareSpec](#v1beta2-hardwarespec). -Workload defines the synthetic traffic model used for every candidate evaluation. When omitted, admission supplies a static shape of ISL 1024, OSL 128 and concurrency 10; an explicit empty workload receives the same shape from the schema. An explicit workload.static: \{\} opts out of that shape, leaving the workload to spec.aisimulate.parameters.workload. spec.aisimulate.parameters.workload overrides the fields it names. +Workload defines the synthetic traffic model used for every candidate evaluation. When omitted, admission supplies a static shape of ISL 1024, OSL 128 and concurrency 10; an explicit empty workload receives the same shape from the schema. An explicit workload.static: \{\} opts out of that shape, leaving the workload to spec.aisimulate.parameters.workload. spec.aisimulate.parameters.workload overrides the fields it names. The default evaluates a fixed concurrency, so it creates neither sweep.max_trials nor a KV-load ratio search. See [WorkloadSpec](#v1beta2-workloadspec). **Validation:** Optional: \{\} @@ -5296,7 +5296,7 @@ See [ObjectiveMode](#objectivemode). -Metric is the scalar target when mode is Optimize. When omitted in Optimize mode, admission defaults it to ThroughputPerGPU. +Metric is the scalar target when mode is Optimize. When omitted in Optimize mode, admission defaults it to Throughput. See [ObjectiveMetric](#objectivemetric). **Validation:** Enum: [Throughput ThroughputPerGPU ThroughputPerUser TTFT E2ELatency Goodput GoodputPerGPU] Optional: \{\} From 8d85b2cb8c6e02d76cdf44e8c16845b8162d8a3e Mon Sep 17 00:00:00 2001 From: devivasudevan <49675305+devivasudevan@users.noreply.github.com> Date: Wed, 21 Oct 2026 22:18:34 +0200 Subject: [PATCH 20/40] feat(operator): require effective AISimulate hardware at v1beta2 admission Validate the lowered input, so hardware given only in spec.aisimulate.parameters counts and a null that deletes discovered hardware does not. Updates validate only when the spec changed. Rewrite the aisimulate package contract to match. Signed-off-by: devivasudevan <49675305+devivasudevan@users.noreply.github.com> --- .../internal/controller/aisimulate/doc.go | 22 ++--- ...hdeploymentrequest_carrier_envtest_test.go | 2 +- .../dynamographdeploymentrequest_v1beta2.go | 42 ++++++++- ...ploymentrequest_validation_envtest_test.go | 86 ++++++++++++++++--- 4 files changed, 125 insertions(+), 27 deletions(-) diff --git a/deploy/operator/internal/controller/aisimulate/doc.go b/deploy/operator/internal/controller/aisimulate/doc.go index 07d398d60e78..b4510da1fc82 100644 --- a/deploy/operator/internal/controller/aisimulate/doc.go +++ b/deploy/operator/internal/controller/aisimulate/doc.go @@ -18,17 +18,19 @@ // Package aisimulate lowers a v1beta2 DynamoGraphDeploymentRequest spec into the native // AISimulate input that a Run executes. // -// Lowering sits below the Request reconciler. Admission calls it as a second validation -// stage, after the typed spec has been defaulted and validated on its own, and the -// reconciler freezes its output into the Run, so both see the same bytes. +// Lowering sits below the Request reconciler. Admission calls Parameters to check that +// spec.aisimulate.parameters is an object, and Lower to check the effective input, for +// example that it names the hardware. The reconciler freezes the output of Lower into the Run, +// so admission and the Run see the same bytes. // -// The typed spec (model, backends, hardware, workload, objective) is first translated -// into the native input, the "underlay". spec.aisimulate.parameters is then applied on -// top as a JSON Merge Patch (RFC 7386): native objects merge recursively, arrays and -// scalars replace the underlay value, and null deletes it. The native input therefore -// wins on every path it names. Finally the derived defaults that no API field carries are -// added: the request count, a bounded trial budget for a KV-load range search, and -// goal.strict_sla. +// Lower works in three steps. The typed spec (model, backends, hardware, workload, objective) +// is first translated into the native input, the "underlay". The derived defaults that no API +// field carries are then added to it: the request count, a bounded trial budget for a +// KV-load range search, and goal.strict_sla when an SLA bound is set. Last, +// spec.aisimulate.parameters is applied on top as a JSON Merge Patch (RFC 7386): native +// objects merge recursively, arrays and scalars replace the value, and null deletes it. The +// parameters are authoritative: they win on every path they name, including the derived +// defaults, and are applied once, last. // // The document stays unstructured (map[string]any) because its schema belongs to the // AISimulate release the Request selects, not to this package. Reads and writes go diff --git a/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_carrier_envtest_test.go b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_carrier_envtest_test.go index 843b415057aa..f8f8eac32574 100644 --- a/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_carrier_envtest_test.go +++ b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_carrier_envtest_test.go @@ -23,7 +23,7 @@ func TestDynamoGraphDeploymentRequestCarrierObjectsStayWritable(t *testing.T) { carrierRoundTrip(t, betaDGDRForAdmission(nil), beta2DGDRForAdmission(nil), "v1beta1") }) t.Run("native v1beta2 written through v1beta1", func(t *testing.T) { - carrierRoundTrip(t, beta2DGDRForAdmission(nil), betaDGDRForAdmission(nil), "v1beta2") + carrierRoundTrip(t, beta2DGDRForAdmission(setBeta2AdmissionHardware), betaDGDRForAdmission(nil), "v1beta2") }) } diff --git a/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_v1beta2.go b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_v1beta2.go index ab5799d779d8..9ac68b3b4491 100644 --- a/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_v1beta2.go +++ b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_v1beta2.go @@ -24,6 +24,7 @@ import ( nvidiacomv1beta2 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta2" "github.com/ai-dynamo/dynamo/deploy/operator/internal/controller/aisimulate" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/util/validation/field" ) @@ -47,12 +48,16 @@ func validateDynamoGraphDeploymentRequestEnvelopeV1beta2( } // validateDynamoGraphDeploymentRequestV1beta2 validates a defaulted v1beta2 Request spec: the -// typed fields, then that spec.aisimulate.parameters is an object. The content of the -// parameters is not validated here; the AISimulate input schema owns it. +// typed fields, that spec.aisimulate.parameters is an object, and that the effective AISimulate +// input names the hardware. The rest of the parameters content is not validated here; the +// AISimulate input schema owns it. Update calls this only when the spec changed, so objects +// stored before this check keep working until they are edited. func validateDynamoGraphDeploymentRequestV1beta2(spec *nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec) field.ErrorList { errs := validateDynamoGraphDeploymentRequestSpecV1beta2(spec, field.NewPath("spec")) if _, err := aisimulate.Parameters(spec); err != nil { errs = append(errs, field.Invalid(parametersPath, "", err.Error())) + } else { + errs = append(errs, validateEffectiveHardwareV1beta2(spec)...) } return errs } @@ -123,3 +128,36 @@ func validateAISimulateSpecV1beta2(spec *nvidiacomv1beta2.AISimulateSpec, fldPat } return nil } + +// validateEffectiveHardwareV1beta2 requires the AISimulate input that a Run would execute to name +// the GPU SKU and budget. The input is the typed spec lowered and then patched by +// spec.aisimulate.parameters, so hardware given only in the parameters satisfies it, and a +// parameters null that deletes discovered hardware does not. Errors are reported at the typed +// spec.hardware.gpu fields, which is where the user fixes them. +func validateEffectiveHardwareV1beta2(spec *nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec) field.ErrorList { + input, err := aisimulate.Lower(spec) + if err != nil { + return field.ErrorList{field.Invalid(parametersPath, "", err.Error())} + } + gpuPath := field.NewPath("spec", "hardware", "gpu") + var errs field.ErrorList + if !effectiveValuePresent(input, aisimulate.SearchSpaceHardwareSKU) { + errs = append(errs, field.Required(gpuPath.Child("sku"), + "the effective AISimulate input has no hardware SKU: set it, enable GPU discovery, or set spec.aisimulate.parameters.search_space.hardware_sku")) + } + if !effectiveValuePresent(input, aisimulate.SearchSpaceGPUBudget) { + errs = append(errs, field.Required(gpuPath.Child("budget"), + "the effective AISimulate input has no GPU budget: set it, enable GPU discovery, or set spec.aisimulate.parameters.search_space.gpu_budget")) + } + return errs +} + +// effectiveValuePresent reports whether input holds a non-null, non-empty value at path. +func effectiveValuePresent(input map[string]any, path []string) bool { + value, found, err := unstructured.NestedFieldNoCopy(input, path...) + if err != nil || !found || value == nil { + return false + } + text, isString := value.(string) + return !isString || text != "" +} diff --git a/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_validation_envtest_test.go b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_validation_envtest_test.go index 29055c68eaf9..339ccde92aa3 100644 --- a/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_validation_envtest_test.go +++ b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_validation_envtest_test.go @@ -117,12 +117,13 @@ func TestDynamoGraphDeploymentRequestValidator_Validate(t *testing.T) { }, { name: "v1beta2 create bypasses legacy v1beta1 validation", - request: beta2DGDRForAdmission(nil), + request: beta2DGDRForAdmission(setBeta2AdmissionHardware), }, { name: "v1beta2 update bypasses legacy v1beta1 validation", - oldRequest: beta2DGDRForAdmission(nil), + oldRequest: beta2DGDRForAdmission(setBeta2AdmissionHardware), request: beta2DGDRForAdmission(func(request *nvidiacomv1beta2.DynamoGraphDeploymentRequest) { + setBeta2AdmissionHardware(request) request.Labels = map[string]string{"updated": "true"} }), }, @@ -574,17 +575,47 @@ func TestDynamoGraphDeploymentRequestValidator_Validate(t *testing.T) { }, }, { - name: "v1beta2 failed discovery leaves the hardware absent", + name: "v1beta2 failed discovery is rejected", request: v1beta2DGDRForAdmission(nil, nil), gpuDiscovery: true, discoveryErr: errors.New("no GPUs found"), + wantWebhook: []string{ + v1beta2SKURequired, + v1beta2BudgetRequired, + }, + }, + { + name: "v1beta2 failed discovery permits hardware given in the parameters", + request: v1beta2DGDRForAdmission(func(spec map[string]any) { + setV1beta2SearchSpace(spec, map[string]any{"hardware_sku": "h100_sxm", "gpu_budget": int64(4)}) + }, nil), + gpuDiscovery: true, + discoveryErr: errors.New("no GPUs found"), check: func(t *testing.T, result *unstructured.Unstructured) { v1beta2Absent(t, result, "spec", "hardware") - if got := v1beta2String(t, result, "spec", "aisimulate", "image"); got != admissionAISimulateImage { - t.Fatalf("aisimulate.image = %q, want %q", got, admissionAISimulateImage) + if got := v1beta2String(t, result, "spec", "aisimulate", "parameters", "search_space", "hardware_sku"); got != "h100_sxm" { + t.Fatalf("parameters.search_space.hardware_sku = %q, want h100_sxm", got) } }, }, + { + name: "v1beta2 failed discovery with only one effective hardware key is rejected at the missing one", + request: v1beta2DGDRForAdmission(func(spec map[string]any) { + setV1beta2SearchSpace(spec, map[string]any{"hardware_sku": "h100_sxm"}) + }, nil), + gpuDiscovery: true, + discoveryErr: errors.New("no GPUs found"), + wantWebhook: []string{v1beta2BudgetRequired}, + }, + { + name: "v1beta2 parameters null that deletes discovered hardware is rejected", + request: v1beta2DGDRForAdmission(func(spec map[string]any) { + setV1beta2SearchSpace(spec, map[string]any{"hardware_sku": nil}) + }, nil), + gpuDiscovery: true, + discovered: discoveredH200, + wantWebhook: []string{v1beta2SKURequired}, + }, { name: "v1beta2 empty workload receives the static model-only default", request: v1beta2DGDRForAdmission(func(spec map[string]any) { @@ -714,12 +745,13 @@ func TestDynamoGraphDeploymentRequestValidator_Validate(t *testing.T) { }, }, { - name: "v1beta2 metadata-only update does not pick up hardware once discovery recovers", - oldRequest: v1beta2DGDRForAdmission(nil, nil), - request: v1beta2DGDRForAdmission(nil, map[string]any{"updated": "true"}), - gpuDiscovery: true, - discoveryErr: errors.New("no GPUs found"), - beforeUpdate: func(*testing.T) { admissionHardware.set(discoveredH200, nil) }, + name: "v1beta2 metadata-only update does not pick up hardware once discovery recovers", + oldRequest: v1beta2DGDRForAdmission(nil, nil), + seedWithoutWebhook: true, // stored before hardware was validated + request: v1beta2DGDRForAdmission(nil, map[string]any{"updated": "true"}), + gpuDiscovery: true, + discoveryErr: errors.New("no GPUs found"), + beforeUpdate: func(*testing.T) { admissionHardware.set(discoveredH200, nil) }, check: func(t *testing.T, result *unstructured.Unstructured) { if result.GetGeneration() != 1 { t.Fatalf("generation = %d, want 1: an update must not rewrite the spec", result.GetGeneration()) @@ -829,7 +861,7 @@ func TestDynamoGraphDeploymentRequestStatusRepresentationMatchesSpec(t *testing. }, { name: "v1alpha1 v1beta2 spec rejects native status", - createObject: beta2DGDRForAdmission(nil), + createObject: beta2DGDRForAdmission(setBeta2AdmissionHardware), statusObject: alphaDGDRForAdmission(nil), status: map[string]any{"state": string(nvidiacomv1alpha1.DGDRStatePending)}, }, @@ -841,13 +873,13 @@ func TestDynamoGraphDeploymentRequestStatusRepresentationMatchesSpec(t *testing. }, { name: "v1beta1 v1beta2 spec rejects native status", - createObject: beta2DGDRForAdmission(nil), + createObject: beta2DGDRForAdmission(setBeta2AdmissionHardware), statusObject: betaDGDRForAdmission(nil), status: map[string]any{"phase": string(nvidiacomv1beta1.DGDRPhasePending)}, }, { name: "v1beta2 native spec rejects v1beta1 status", - createObject: beta2DGDRForAdmission(nil), + createObject: beta2DGDRForAdmission(setBeta2AdmissionHardware), statusObject: beta2DGDRForAdmission(nil), status: map[string]any{"v1beta1": map[string]any{"phase": string(nvidiacomv1beta1.DGDRPhasePending)}}, }, @@ -898,6 +930,7 @@ func TestDynamoGraphDeploymentRequestV1Beta2EmptyStatusSurvivesV1Alpha1(t *testi warnings := &warningRecorder{} beta2Fixture := beta2DGDRForAdmission(func(request *nvidiacomv1beta2.DynamoGraphDeploymentRequest) { request.Name = "empty-status-alpha-bridge" + setBeta2AdmissionHardware(request) }) beta2Client := newAdmissionResourceClient(t, env, beta2Fixture, "", warnings) object, _ := admissionObject(t, beta2Fixture, env.Namespace(), nil) @@ -1141,3 +1174,28 @@ func expectV1beta2AdmissionDefaults(t *testing.T, obj *unstructured.Unstructured t.Fatalf("hardware.gpu.budget = %v, want 8", got) } } + +// setV1beta2SearchSpace sets spec.aisimulate.parameters.search_space to searchSpace, replacing +// any parameters the fixture carried. +func setV1beta2SearchSpace(spec map[string]any, searchSpace map[string]any) { + aisimulate, _ := spec["aisimulate"].(map[string]any) + if aisimulate == nil { + aisimulate = map[string]any{} + spec["aisimulate"] = aisimulate + } + aisimulate["parameters"] = map[string]any{"search_space": searchSpace} +} + +const ( + v1beta2SKURequired = "spec.hardware.gpu.sku: Required value: the effective AISimulate input has no hardware SKU: set it, enable GPU discovery, or set spec.aisimulate.parameters.search_space.hardware_sku" + v1beta2BudgetRequired = "spec.hardware.gpu.budget: Required value: the effective AISimulate input has no GPU budget: set it, enable GPU discovery, or set spec.aisimulate.parameters.search_space.gpu_budget" +) + +// setBeta2AdmissionHardware names the hardware, as admission requires when GPU discovery finds +// none. Tests that are not about hardware use it so their requests are valid. +func setBeta2AdmissionHardware(request *nvidiacomv1beta2.DynamoGraphDeploymentRequest) { + sku, budget := "h100_sxm", int32(4) + request.Spec.Hardware = &nvidiacomv1beta2.HardwareSpec{ + GPU: &nvidiacomv1beta2.GPUHardwareSpec{SKU: &sku, Budget: &budget}, + } +} From e6f68918f394fbc3339c3a0aa287bc5db4479c3a Mon Sep 17 00:00:00 2001 From: devivasudevan <49675305+devivasudevan@users.noreply.github.com> Date: Thu, 22 Oct 2026 09:32:27 +0200 Subject: [PATCH 21/40] fix(operator): address review: lint, DCGM error reason, discovery deadline Split the flag description (lll). Drop the constant kind parameter of the test helper (unparam). Match the lowercased error text for the DCGM-not-enabled reason, which never matched. Give the DCGM phase of GPU discovery half of the remaining deadline so the node-label fallback keeps time to run. Signed-off-by: devivasudevan <49675305+devivasudevan@users.noreply.github.com> --- deploy/operator/cmd/main.go | 3 +- .../internal/gpu/dcgm_phase_context_test.go | 38 +++++++++++++++++++ .../internal/gpu/hardware_discoverer.go | 24 +++++++++++- .../internal/webhook/request_version_test.go | 24 ++++++------ 4 files changed, 75 insertions(+), 14 deletions(-) create mode 100644 deploy/operator/internal/gpu/dcgm_phase_context_test.go diff --git a/deploy/operator/cmd/main.go b/deploy/operator/cmd/main.go index 8ec67cd11115..ce8c7a2503d6 100644 --- a/deploy/operator/cmd/main.go +++ b/deploy/operator/cmd/main.go @@ -178,7 +178,8 @@ func main() { flag.StringVar(&dgdrDefaultImage, "dgdr-default-image", "", "Default DGDR profiler image, put into DGDR spec.image when unset; empty derives dynamo-planner:") flag.StringVar(&dgdrAISimulateDefaultImage, "dgdr-aisimulate-default-image", "", - "AISimulate runner image, put into v1beta2 DGDR spec.aisimulate.image when unset; empty leaves the image unset and the Request is rejected") + "AISimulate runner image, put into v1beta2 DGDR spec.aisimulate.image when unset; "+ + "empty leaves the image unset and the Request is rejected") opts := zap.Options{ Development: true, } diff --git a/deploy/operator/internal/gpu/dcgm_phase_context_test.go b/deploy/operator/internal/gpu/dcgm_phase_context_test.go new file mode 100644 index 000000000000..aea20e9362e2 --- /dev/null +++ b/deploy/operator/internal/gpu/dcgm_phase_context_test.go @@ -0,0 +1,38 @@ +package gpu + +import ( + "context" + "testing" + "time" +) + +func TestDCGMPhaseContextReservesTimeForTheFallback(t *testing.T) { + outer, cancelOuter := context.WithTimeout(t.Context(), 4*time.Second) + defer cancelOuter() + outerDeadline, _ := outer.Deadline() + + t.Log("With the fallback enabled the DCGM phase ends well before the outer deadline") + phase, cancel := dcgmPhaseContext(outer, true) + defer cancel() + phaseDeadline, ok := phase.Deadline() + if !ok { + t.Fatal("phase context has no deadline") + } + if remaining := outerDeadline.Sub(phaseDeadline); remaining < time.Second { + t.Fatalf("only %v is left for the fallback, want at least 1s", remaining) + } + + t.Log("Without the fallback the whole context is used") + same, cancelSame := dcgmPhaseContext(outer, false) + defer cancelSame() + if same != outer { + t.Fatal("context was narrowed although there is no fallback to reserve time for") + } + + t.Log("A context without a deadline is left alone") + bare, cancelBare := dcgmPhaseContext(t.Context(), true) + defer cancelBare() + if _, hasDeadline := bare.Deadline(); hasDeadline { + t.Fatal("a deadline was invented for a context that had none") + } +} diff --git a/deploy/operator/internal/gpu/hardware_discoverer.go b/deploy/operator/internal/gpu/hardware_discoverer.go index 6f060a22e291..4aad267e994b 100644 --- a/deploy/operator/internal/gpu/hardware_discoverer.go +++ b/deploy/operator/internal/gpu/hardware_discoverer.go @@ -22,6 +22,7 @@ import ( "errors" "fmt" "strings" + "time" nvidiacomv1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" "sigs.k8s.io/controller-runtime/pkg/client" @@ -53,7 +54,10 @@ func (d *Discoverer) Discover(ctx context.Context, filterSKU nvidiacomv1beta1.GP var dcgmErr error if d.DCGM != nil { - info, err := d.DCGM.DiscoverGPUsFromDCGMFiltered(ctx, d.Reader, d.Cache, filterSKU) + // A slow exporter must not leave the node-label fallback with an expired context. + dcgmCtx, cancel := dcgmPhaseContext(ctx, d.NodeLabels != nil && d.NodeLabels()) + info, err := d.DCGM.DiscoverGPUsFromDCGMFiltered(dcgmCtx, d.Reader, d.Cache, filterSKU) + cancel() if err == nil { return info, nil } @@ -127,7 +131,7 @@ func DiscoveryFailureReason(err error) string { return "failed to parse dcgm Prometheus metrics (invalid format)" case strings.Contains(errMsg, "no gpus detected"): return "no GPUs detected in dcgm metrics (GPU model or metrics missing)" - case strings.Contains(errMsg, "dcgm is not enabled in the GPU Operator"): + case strings.Contains(errMsg, "dcgm is not enabled in the gpu operator"): return "DCGM is not enabled in the GPU Operator (check GPU Operator configuration and permissions)" case strings.Contains(errMsg, "failed to scrape any dcgm exporter pod"): return "failed to scrape any dcgm exporter pod (check DCGM exporter pod status and network connectivity)" @@ -138,3 +142,19 @@ func DiscoveryFailureReason(err error) string { } return unknownReason } + +// dcgmPhaseContext bounds the DCGM phase of discovery. When the node-label fallback is enabled +// and ctx has a deadline, the DCGM phase gets half of what remains, so a slow exporter cannot +// consume the whole admission budget and leave the fallback with an expired context. Otherwise +// ctx is returned unchanged. The returned cancel function must be called. +func dcgmPhaseContext(ctx context.Context, reserveForFallback bool) (context.Context, context.CancelFunc) { + deadline, ok := ctx.Deadline() + if !ok || !reserveForFallback { + return ctx, func() {} + } + remaining := time.Until(deadline) + if remaining <= 0 { + return ctx, func() {} + } + return context.WithTimeout(ctx, remaining/2) +} diff --git a/deploy/operator/internal/webhook/request_version_test.go b/deploy/operator/internal/webhook/request_version_test.go index df630a49f1af..3a6680142eda 100644 --- a/deploy/operator/internal/webhook/request_version_test.go +++ b/deploy/operator/internal/webhook/request_version_test.go @@ -25,9 +25,11 @@ import ( "sigs.k8s.io/controller-runtime/pkg/webhook/admission" ) -func requestWithKinds(kind string, requestKind string) admission.Request { +// requestWithKinds returns a request whose stored kind is v1beta1, the version every webhook +// receives after conversion, and whose original request kind is requestKind (empty for none). +func requestWithKinds(requestKind string) admission.Request { req := admission.Request{AdmissionRequest: admissionv1.AdmissionRequest{ - Kind: metav1.GroupVersionKind{Group: "nvidia.com", Version: kind, Kind: "DynamoGraphDeploymentRequest"}, + Kind: metav1.GroupVersionKind{Group: "nvidia.com", Version: "v1beta1", Kind: "DynamoGraphDeploymentRequest"}, }} if requestKind != "" { req.RequestKind = &metav1.GroupVersionKind{Group: "nvidia.com", Version: requestKind, Kind: "DynamoGraphDeploymentRequest"} @@ -41,9 +43,9 @@ func TestRequestedVersion(t *testing.T) { req admission.Request want string }{ - {"no conversion uses Kind", requestWithKinds("v1beta1", ""), "v1beta1"}, - {"RequestKind wins after conversion", requestWithKinds("v1beta1", "v1beta2"), "v1beta2"}, - {"RequestKind wins over a matching Kind", requestWithKinds("v1beta1", "v1alpha1"), "v1alpha1"}, + {"no conversion uses Kind", requestWithKinds(""), "v1beta1"}, + {"RequestKind wins after conversion", requestWithKinds("v1beta2"), "v1beta2"}, + {"RequestKind wins over a matching Kind", requestWithKinds("v1alpha1"), "v1alpha1"}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -61,12 +63,12 @@ func TestRequestIsNative(t *testing.T) { carrier bool want bool }{ - {"native v1beta2 create", requestWithKinds("v1beta1", "v1beta2"), true, true}, - {"native v1beta1 create", requestWithKinds("v1beta1", ""), false, true}, - {"native v1alpha1 create", requestWithKinds("v1beta1", "v1alpha1"), false, true}, - {"carrier written through v1beta1", requestWithKinds("v1beta1", ""), true, false}, - {"carrier written through v1alpha1", requestWithKinds("v1beta1", "v1alpha1"), true, false}, - {"v1beta1 payload written through v1beta2", requestWithKinds("v1beta1", "v1beta2"), false, false}, + {"native v1beta2 create", requestWithKinds("v1beta2"), true, true}, + {"native v1beta1 create", requestWithKinds(""), false, true}, + {"native v1alpha1 create", requestWithKinds("v1alpha1"), false, true}, + {"carrier written through v1beta1", requestWithKinds(""), true, false}, + {"carrier written through v1alpha1", requestWithKinds("v1alpha1"), true, false}, + {"v1beta1 payload written through v1beta2", requestWithKinds("v1beta2"), false, false}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { From 629636cbf6291c71e1ba6d86886d78a40cc6cf31 Mon Sep 17 00:00:00 2001 From: devivasudevan <49675305+devivasudevan@users.noreply.github.com> Date: Thu, 22 Oct 2026 11:11:49 +0200 Subject: [PATCH 22/40] fix(operator): ask the node-label fallback only when the context has a deadline The DCGM phase reserved time for the fallback by calling NodeLabels() before DCGM ran. Callers without a deadline, such as the v1beta1 reconciler, never needed that and could hit a callback that was not configured for them. Evaluate it only when there is a deadline to share, and test that it is not asked otherwise. Signed-off-by: devivasudevan <49675305+devivasudevan@users.noreply.github.com> --- .../operator/internal/gpu/dcgm_phase_context_test.go | 9 ++++++--- deploy/operator/internal/gpu/hardware_discoverer.go | 12 +++++++----- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/deploy/operator/internal/gpu/dcgm_phase_context_test.go b/deploy/operator/internal/gpu/dcgm_phase_context_test.go index aea20e9362e2..ba4a86fe6fb6 100644 --- a/deploy/operator/internal/gpu/dcgm_phase_context_test.go +++ b/deploy/operator/internal/gpu/dcgm_phase_context_test.go @@ -12,7 +12,7 @@ func TestDCGMPhaseContextReservesTimeForTheFallback(t *testing.T) { outerDeadline, _ := outer.Deadline() t.Log("With the fallback enabled the DCGM phase ends well before the outer deadline") - phase, cancel := dcgmPhaseContext(outer, true) + phase, cancel := dcgmPhaseContext(outer, func() bool { return true }) defer cancel() phaseDeadline, ok := phase.Deadline() if !ok { @@ -23,14 +23,17 @@ func TestDCGMPhaseContextReservesTimeForTheFallback(t *testing.T) { } t.Log("Without the fallback the whole context is used") - same, cancelSame := dcgmPhaseContext(outer, false) + same, cancelSame := dcgmPhaseContext(outer, func() bool { return false }) defer cancelSame() if same != outer { t.Fatal("context was narrowed although there is no fallback to reserve time for") } t.Log("A context without a deadline is left alone") - bare, cancelBare := dcgmPhaseContext(t.Context(), true) + bare, cancelBare := dcgmPhaseContext(t.Context(), func() bool { + t.Fatal("the fallback callback was asked although the context has no deadline") + return true + }) defer cancelBare() if _, hasDeadline := bare.Deadline(); hasDeadline { t.Fatal("a deadline was invented for a context that had none") diff --git a/deploy/operator/internal/gpu/hardware_discoverer.go b/deploy/operator/internal/gpu/hardware_discoverer.go index 4aad267e994b..988ffb428d72 100644 --- a/deploy/operator/internal/gpu/hardware_discoverer.go +++ b/deploy/operator/internal/gpu/hardware_discoverer.go @@ -55,7 +55,7 @@ func (d *Discoverer) Discover(ctx context.Context, filterSKU nvidiacomv1beta1.GP var dcgmErr error if d.DCGM != nil { // A slow exporter must not leave the node-label fallback with an expired context. - dcgmCtx, cancel := dcgmPhaseContext(ctx, d.NodeLabels != nil && d.NodeLabels()) + dcgmCtx, cancel := dcgmPhaseContext(ctx, d.NodeLabels) info, err := d.DCGM.DiscoverGPUsFromDCGMFiltered(dcgmCtx, d.Reader, d.Cache, filterSKU) cancel() if err == nil { @@ -143,13 +143,15 @@ func DiscoveryFailureReason(err error) string { return unknownReason } -// dcgmPhaseContext bounds the DCGM phase of discovery. When the node-label fallback is enabled -// and ctx has a deadline, the DCGM phase gets half of what remains, so a slow exporter cannot +// dcgmPhaseContext bounds the DCGM phase of discovery. When ctx has a deadline and fallbackEnabled +// reports that the node-label fallback is enabled, the DCGM phase gets half of what remains, so a slow exporter cannot // consume the whole admission budget and leave the fallback with an expired context. Otherwise // ctx is returned unchanged. The returned cancel function must be called. -func dcgmPhaseContext(ctx context.Context, reserveForFallback bool) (context.Context, context.CancelFunc) { +func dcgmPhaseContext(ctx context.Context, fallbackEnabled func() bool) (context.Context, context.CancelFunc) { deadline, ok := ctx.Deadline() - if !ok || !reserveForFallback { + // fallbackEnabled is only asked when there is a deadline to share: without one nothing is reserved, + // and the callback may depend on state that a caller without a deadline never configured. + if !ok || fallbackEnabled == nil || !fallbackEnabled() { return ctx, func() {} } remaining := time.Until(deadline) From f2c2a7e9f0a2cc4548bcf74993f46fe6cfb38c73 Mon Sep 17 00:00:00 2001 From: "Dr. Stefan Schimanski" Date: Thu, 22 Oct 2026 10:03:39 +0200 Subject: [PATCH 23/40] operator/admission: derive default AISimulate image Signed-off-by: Dr. Stefan Schimanski --- .../platform/components/operator/values.yaml | 2 +- .../dynamographdeploymentrequest_types.go | 2 +- deploy/operator/cmd/main.go | 2 +- .../dynamographdeploymentrequest_handler.go | 16 ++++---- ...namographdeploymentrequest_handler_test.go | 7 ++-- ...namographdeploymentrequest_routing_test.go | 37 ++++++++++++++----- .../dynamographdeploymentrequest_v1beta2.go | 2 +- ...ographdeploymentrequest_v1beta2_helpers.go | 5 +-- .../webhook/validation/suite_envtest_test.go | 5 +-- .../additional-resources/api-reference-k8s.md | 2 +- .../kubernetes-api/full-api-reference.mdx | 2 +- 11 files changed, 48 insertions(+), 34 deletions(-) diff --git a/deploy/helm/charts/platform/components/operator/values.yaml b/deploy/helm/charts/platform/components/operator/values.yaml index bbc3df59e7c7..36f69db1aba7 100644 --- a/deploy/helm/charts/platform/components/operator/values.yaml +++ b/deploy/helm/charts/platform/components/operator/values.yaml @@ -116,7 +116,7 @@ controllerManager: # releases. dgdrDefaultImage: "" # AISimulate runner image put into v1beta2 DGDR spec.aisimulate.image when unset. - # Empty leaves it unset, and v1beta2 Requests without an image are rejected. + # Empty derives nvcr.io/nvidia/ai-dynamo/dynamo-planner:. dgdrAISimulateDefaultImage: "" resources: limits: diff --git a/deploy/operator/api/v1beta2/dynamographdeploymentrequest_types.go b/deploy/operator/api/v1beta2/dynamographdeploymentrequest_types.go index 703825595547..e20fc4d782d9 100644 --- a/deploy/operator/api/v1beta2/dynamographdeploymentrequest_types.go +++ b/deploy/operator/api/v1beta2/dynamographdeploymentrequest_types.go @@ -185,7 +185,7 @@ type ObjectiveSpec struct { // AISimulateSpec selects the AISimulate execution artifact and supplies native input. type AISimulateSpec struct { // Image selects a Dynamo-compatible AISimulate runner. When omitted, admission - // persists the operator's configured image. + // persists the configured override or a version-matched Dynamo planner image. // +optional // +kubebuilder:validation:MinLength=1 Image string `json:"image,omitempty"` diff --git a/deploy/operator/cmd/main.go b/deploy/operator/cmd/main.go index ce8c7a2503d6..3c762841c7a2 100644 --- a/deploy/operator/cmd/main.go +++ b/deploy/operator/cmd/main.go @@ -179,7 +179,7 @@ func main() { "Default DGDR profiler image, put into DGDR spec.image when unset; empty derives dynamo-planner:") flag.StringVar(&dgdrAISimulateDefaultImage, "dgdr-aisimulate-default-image", "", "AISimulate runner image, put into v1beta2 DGDR spec.aisimulate.image when unset; "+ - "empty leaves the image unset and the Request is rejected") + "empty derives dynamo-planner:") opts := zap.Options{ Development: true, } diff --git a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_handler.go b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_handler.go index eb6ef7dca92e..8826650e166c 100644 --- a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_handler.go +++ b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_handler.go @@ -76,6 +76,7 @@ func NewDGDRDefaulter(operatorVersion, defaultImage string) *DGDRDefaulter { // WithV1beta2 sets the defaults of v1beta2 Requests and returns the defaulter. func (d *DGDRDefaulter) WithV1beta2(options DGDRV1beta2Options) *DGDRDefaulter { + options.AISimulateImage = defaultPlannerImageFor(d.OperatorVersion, options.AISimulateImage) d.v1beta2 = options return d } @@ -127,7 +128,7 @@ func (d *DGDRDefaulter) Default(ctx context.Context, obj runtime.Object) error { // Image derivation remains create-only so updates do not rewrite an // intentionally cleared image. if req.Operation == admissionv1.Create && dgdr.Spec.Image == "" { - if img := d.defaultImageFor(); img != "" { + if img := defaultPlannerImageFor(d.OperatorVersion, d.DefaultImage); img != "" { dgdr.Spec.Image = img logger.Info("defaulted spec.image from operator version", "name", dgdr.Name, @@ -140,14 +141,13 @@ func (d *DGDRDefaulter) Default(ctx context.Context, obj runtime.Object) error { return nil } -// defaultImageFor returns the profiler image for spec.image: DefaultImage when -// set, else the derived image with a canonical semver tag, or an empty string -// when the operator version cannot be parsed. -func (d *DGDRDefaulter) defaultImageFor() string { - if d.DefaultImage != "" { - return d.DefaultImage +// defaultPlannerImageFor returns explicitImage when set, else the planner image with a +// canonical operator-version tag, or an empty string when the version cannot be parsed. +func defaultPlannerImageFor(operatorVersion, explicitImage string) string { + if explicitImage != "" { + return explicitImage } - version, err := semver.NewVersion(d.OperatorVersion) + version, err := semver.NewVersion(operatorVersion) if err != nil { return "" } diff --git a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_handler_test.go b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_handler_test.go index 4f2c35382611..48fafb9ead7c 100644 --- a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_handler_test.go +++ b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_handler_test.go @@ -28,7 +28,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/webhook/admission" ) -func TestDGDRDefaulter_defaultImageFor(t *testing.T) { +func TestDefaultPlannerImageFor(t *testing.T) { // Note: the default planner image is only published starting from Dynamo 1.1.0, // so these tests use 1.1.0 as the earliest known valid version. tests := []struct { @@ -77,10 +77,9 @@ func TestDGDRDefaulter_defaultImageFor(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - d := NewDGDRDefaulter(tt.operatorVersion, tt.defaultImage) - got := d.defaultImageFor() + got := defaultPlannerImageFor(tt.operatorVersion, tt.defaultImage) if got != tt.expectedImage { - t.Errorf("defaultImageFor() = %q, want %q", got, tt.expectedImage) + t.Errorf("defaultPlannerImageFor() = %q, want %q", got, tt.expectedImage) } }) } diff --git a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_routing_test.go b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_routing_test.go index 7dbb6acd8950..416fe004f280 100644 --- a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_routing_test.go +++ b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_routing_test.go @@ -67,17 +67,34 @@ func aisimulateImageOf(t *testing.T, raw []byte) string { } func TestDGDRDefaulter_RoutesByRequestedVersion(t *testing.T) { - d := NewDGDRDefaulter("1.1.0", "").WithV1beta2(DGDRV1beta2Options{AISimulateImage: "aisim:1"}) + for _, tt := range []struct { + name string + options DGDRV1beta2Options + want string + }{ + { + name: "an omitted image is derived from the operator version", + want: "nvcr.io/nvidia/ai-dynamo/dynamo-planner:1.6.0", + }, + { + name: "an explicit default image overrides derivation", + options: DGDRV1beta2Options{AISimulateImage: "registry.example/aisimulate:custom"}, + want: "registry.example/aisimulate:custom", + }, + } { + t.Run(tt.name, func(t *testing.T) { + d := NewDGDRDefaulter("1.6.0", "").WithV1beta2(tt.options) + dgdr := carrierRequest() + if err := d.Default(ctxForRequest(admissionv1.Create, "v1beta2"), dgdr); err != nil { + t.Fatalf("Default() error = %v", err) + } + if got := aisimulateImageOf(t, dgdr.Spec.V1Beta2.Raw); got != tt.want { + t.Fatalf("aisimulate.image = %q, want %q", got, tt.want) + } + }) + } - t.Run("a native v1beta2 request is defaulted from its carrier", func(t *testing.T) { - dgdr := carrierRequest() - if err := d.Default(ctxForRequest(admissionv1.Create, "v1beta2"), dgdr); err != nil { - t.Fatalf("Default() error = %v", err) - } - if got := aisimulateImageOf(t, dgdr.Spec.V1Beta2.Raw); got != "aisim:1" { - t.Fatalf("aisimulate.image = %q, want aisim:1", got) - } - }) + d := NewDGDRDefaulter("1.6.0", "").WithV1beta2(DGDRV1beta2Options{}) t.Run("RequestKind decides, not the converted Kind", func(t *testing.T) { dgdr := carrierRequest() diff --git a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2.go b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2.go index e8a6eebbc89d..ae9014d1e006 100644 --- a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2.go +++ b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2.go @@ -95,7 +95,7 @@ func defaultDynamoGraphDeploymentRequestV1beta2( type dynamoGraphDeploymentRequestV1beta2DefaultOptions struct { // hardware is the GPU identity and budget discovered from the cluster. hardware hardwareDefaultsV1beta2 - // aisimulateImage is the operator's configured AISimulate image. + // aisimulateImage is the operator's resolved AISimulate image. aisimulateImage string } diff --git a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_helpers.go b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_helpers.go index 31a9996534ea..56271a21b82b 100644 --- a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_helpers.go +++ b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_helpers.go @@ -57,9 +57,8 @@ type HardwareDiscoverer interface { // DGDRV1beta2Options configures the defaults of v1beta2 Requests. type DGDRV1beta2Options struct { - // AISimulateImage is the operator's AISimulate runner image, persisted into - // spec.aisimulate.image when the caller omits it. Empty persists nothing, and - // validation then requires the image to be spelled out. + // AISimulateImage overrides the version-matched Dynamo planner image persisted + // into spec.aisimulate.image when the caller omits it. AISimulateImage string // Hardware discovers the cluster's GPUs. Nil disables hardware defaulting, and // validation then requires the hardware to be spelled out. diff --git a/deploy/operator/internal/webhook/validation/suite_envtest_test.go b/deploy/operator/internal/webhook/validation/suite_envtest_test.go index c09e02c99d5b..0886a298281e 100644 --- a/deploy/operator/internal/webhook/validation/suite_envtest_test.go +++ b/deploy/operator/internal/webhook/validation/suite_envtest_test.go @@ -44,7 +44,7 @@ const ( frontendImage150 = "registry.example/dynamo-frontend:1.5.0" legacyEPPImage140 = "registry.example/epp-image:1.4.0" legacySeedUsername = "operatorenv-legacy-seeder" - admissionAISimulateImage = "registry.example/aisimulate:1.2.0" + admissionAISimulateImage = "nvcr.io/nvidia/ai-dynamo/dynamo-planner:1.6.0" ) func groveProviderOverride(target, value string) *nvidiacomv1beta1.ProviderOverride { @@ -97,8 +97,7 @@ func setupAdmissionWebhooks(mgr ctrl.Manager, opts operatorenv.WebhookSetupOptio OperatorPrincipal: opts.OperatorPrincipal, Gate: admissionGate, DGDRV1beta2: webhookdefaulting.DGDRV1beta2Options{ - AISimulateImage: admissionAISimulateImage, - Hardware: admissionHardware, + Hardware: admissionHardware, }, }) } diff --git a/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md b/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md index 2fb7313d7700..899252723888 100644 --- a/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md +++ b/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md @@ -3242,7 +3242,7 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | -| `image` _string_ | Image selects a Dynamo-compatible AISimulate runner. When omitted, admission
persists the operator's configured image. | | MinLength: 1
Optional: \{\}
| +| `image` _string_ | Image selects a Dynamo-compatible AISimulate runner. When omitted, admission
persists the configured override or a version-matched Dynamo planner image. | | MinLength: 1
Optional: \{\}
| | `versionOverride` _string_ | VersionOverride declares the AISimulate compatibility version as three numeric parts
(for example 0.13.0), not a semver version: no v prefix, pre-release or build suffix. It is required
when the image tag is not a version and wins over a version in the tag. | | MinLength: 1
Pattern: `^(0\|[1-9][0-9]\{0,3\})\.(0\|[1-9][0-9]\{0,3\})\.(0\|[1-9][0-9]\{0,3\})$`
Optional: \{\}
| | `parameters` _[RawExtension](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#rawextension-runtime-pkg)_ | Parameters contains native AISimulate input. Field names and nesting use
the upstream AISimulate JSON contract, including snake_case names. | | Type: object
Optional: \{\}
| diff --git a/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx b/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx index dfad335c1aff..019faa3c58ee 100644 --- a/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx +++ b/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx @@ -4531,7 +4531,7 @@ AISimulateSpec selects the AISimulate execution artifact and supplies native inp **Appears in:** [DynamoGraphDeploymentRequestNativeSpec](#v1beta2-dynamographdeploymentrequestnativespec), [DynamoGraphDeploymentRequestSpec](#v1beta2-dynamographdeploymentrequestspec), [DynamoGraphDeploymentRunSpec](#dynamographdeploymentrunspec), [NativeSearchInput](#nativesearchinput) -Image selects a Dynamo-compatible AISimulate runner. When omitted, admission persists the operator's configured image. +Image selects a Dynamo-compatible AISimulate runner. When omitted, admission persists the configured override or a version-matched Dynamo planner image. **Validation:** MinLength: 1 Optional: \{\} From d18a958ef160bba1f83a81cbd9af29a7d9851c15 Mon Sep 17 00:00:00 2001 From: devivasudevan <49675305+devivasudevan@users.noreply.github.com> Date: Thu, 22 Oct 2026 10:58:29 +0200 Subject: [PATCH 24/40] feat(operator): add the v1beta2 DGDR Request controller Create one immutable Run per Request generation, with the effective AISimulate input frozen into aisimulate.parameters and the version resolved. Wait for an active Run before starting the next, record the active and the five most recent terminal Runs in the status, and register the controller only when --dgdr-v1beta2-enabled is set and the cluster serves the v1beta2 APIs. Signed-off-by: devivasudevan <49675305+devivasudevan@users.noreply.github.com> --- deploy/helm/charts/platform/README.md | 2 +- .../components/operator/files/role.yaml | 10 + .../operator/templates/deployment.yaml | 1 + .../platform/components/operator/values.yaml | 4 +- ...namespace_restriction_deployment_test.yaml | 22 ++ deploy/helm/charts/platform/values.yaml | 5 +- deploy/operator/cmd/main.go | 10 + ...aphdeploymentrequest_v1beta2_controller.go | 332 ++++++++++++++++++ ...request_v1beta2_controller_envtest_test.go | 122 +++++++ ...ploymentrequest_v1beta2_controller_test.go | 298 ++++++++++++++++ deploy/operator/internal/controller/setup.go | 16 + 11 files changed, 818 insertions(+), 4 deletions(-) create mode 100644 deploy/operator/internal/controller/dynamographdeploymentrequest_v1beta2_controller.go create mode 100644 deploy/operator/internal/controller/dynamographdeploymentrequest_v1beta2_controller_envtest_test.go create mode 100644 deploy/operator/internal/controller/dynamographdeploymentrequest_v1beta2_controller_test.go diff --git a/deploy/helm/charts/platform/README.md b/deploy/helm/charts/platform/README.md index 28084f1829be..eb9641c2bd20 100644 --- a/deploy/helm/charts/platform/README.md +++ b/deploy/helm/charts/platform/README.md @@ -454,7 +454,7 @@ Kubernetes: `>=1.30.0-0` | dynamo-operator.grovePreflight.deploymentName | string | `""` | External Grove Deployment name. Requires namespace; empty discovers by container name. | | dynamo-operator.enabled | bool | `true` | Whether to enable the Dynamo Kubernetes operator deployment | | dynamo-operator.upgradeCRD | bool | `true` | Whether the cluster-wide operator applies CRDs from its image through the crd-apply init container. When false, the chart installs no CRDs; apply them separately before starting a cluster-wide operator. Namespace-restricted installations must set this to false and use the CRDs managed by the cluster-wide operator. | -| dynamo-operator.dgdrV1Beta2.enabled | bool | `false` | EXPERIMENTAL, DEVELOPMENT CLUSTERS ONLY: Serve the unfinished DGDR v1beta2 Request API with conversion and install the Run and Candidate APIs. Run and Candidate objects are persisted in etcd; incompatible schema changes may make them unreadable or block future CRD upgrades. The operator does not reconcile v1beta2 requests. | +| dynamo-operator.dgdrV1Beta2.enabled | bool | `false` | EXPERIMENTAL, DEVELOPMENT CLUSTERS ONLY: Serve the unfinished DGDR v1beta2 Request API with conversion and install the Run and Candidate APIs. Run and Candidate objects are persisted in etcd; incompatible schema changes may make them unreadable or block future CRD upgrades. When true, the operator also reconciles v1beta2 Requests into Runs; the controller that executes a Run is not part of the operator yet. | | dynamo-operator.natsAddr | string | `""` | NATS server address for operator communication. When empty, the operator uses bundled NATS only if global.nats.install=true; otherwise NATS is not configured. Format: `nats://hostname:4222` | | dynamo-operator.etcdAddr | string | `""` | etcd server address for an external etcd instance. Only needed when using external etcd without the bundled subchart. Format: `http://hostname:2379` or `https://hostname:2379` | | dynamo-operator.modelExpressURL | string | `""` | URL for the Model Express server if not deployed by this helm chart. This is ignored if Model Express server is installed by this helm chart (global.model-express.enabled is true). | diff --git a/deploy/helm/charts/platform/components/operator/files/role.yaml b/deploy/helm/charts/platform/components/operator/files/role.yaml index 1ec01f3f9264..bc0bcfaa36b8 100644 --- a/deploy/helm/charts/platform/components/operator/files/role.yaml +++ b/deploy/helm/charts/platform/components/operator/files/role.yaml @@ -304,6 +304,16 @@ rules: - get - patch - update +- apiGroups: + - nvidia.com + resources: + - dynamographdeploymentruns + verbs: + - create + - delete + - get + - list + - watch - apiGroups: - nvidia.com resources: diff --git a/deploy/helm/charts/platform/components/operator/templates/deployment.yaml b/deploy/helm/charts/platform/components/operator/templates/deployment.yaml index adb853253277..241a98346a27 100644 --- a/deploy/helm/charts/platform/components/operator/templates/deployment.yaml +++ b/deploy/helm/charts/platform/components/operator/templates/deployment.yaml @@ -95,6 +95,7 @@ spec: {{- with .Values.controllerManager.manager.dgdrAISimulateDefaultImage }} - {{ printf "--dgdr-aisimulate-default-image=%s" . | quote }} {{- end }} + - --dgdr-v1beta2-enabled={{ .Values.dgdrV1Beta2.enabled }} command: - /manager env: diff --git a/deploy/helm/charts/platform/components/operator/values.yaml b/deploy/helm/charts/platform/components/operator/values.yaml index 36f69db1aba7..a3ac9a0d7da5 100644 --- a/deploy/helm/charts/platform/components/operator/values.yaml +++ b/deploy/helm/charts/platform/components/operator/values.yaml @@ -36,7 +36,9 @@ dgdrV1Beta2: # Candidate objects are persisted in etcd; incompatible schema changes may make # them unreadable or block future CRD upgrades. When false, the CRD init container # installs the legacy Request schemas without v1beta2 conversion envelopes and - # omits Run and Candidate. The operator does not reconcile v1beta2 requests. + # omits Run and Candidate. When true, the operator also reconciles v1beta2 + # Requests into Runs; the controller that executes a Run is not part of the + # operator yet. enabled: false # Environment variables to pass to operator Deployment. diff --git a/deploy/helm/charts/platform/tests/namespace_restriction_deployment_test.yaml b/deploy/helm/charts/platform/tests/namespace_restriction_deployment_test.yaml index 41789e477c6b..d9855f666593 100644 --- a/deploy/helm/charts/platform/tests/namespace_restriction_deployment_test.yaml +++ b/deploy/helm/charts/platform/tests/namespace_restriction_deployment_test.yaml @@ -93,3 +93,25 @@ tests: asserts: - failedTemplate: errorPattern: "webhook.namespaceSelector must be empty" + + - it: passes the DGDR v1beta2 switch to the manager + template: charts/dynamo-operator/templates/deployment.yaml + asserts: + - contains: + path: spec.template.spec.containers[0].args + content: --dgdr-v1beta2-enabled=false + documentSelector: + path: kind + value: Deployment + + - it: enables the DGDR v1beta2 controller in the manager + template: charts/dynamo-operator/templates/deployment.yaml + set: + dynamo-operator.dgdrV1Beta2.enabled: true + asserts: + - contains: + path: spec.template.spec.containers[0].args + content: --dgdr-v1beta2-enabled=true + documentSelector: + path: kind + value: Deployment diff --git a/deploy/helm/charts/platform/values.yaml b/deploy/helm/charts/platform/values.yaml index 13c14c9c058f..fe6daeec961b 100644 --- a/deploy/helm/charts/platform/values.yaml +++ b/deploy/helm/charts/platform/values.yaml @@ -97,8 +97,9 @@ dynamo-operator: # -- EXPERIMENTAL, DEVELOPMENT CLUSTERS ONLY: Serve the unfinished DGDR v1beta2 # Request API with conversion and install the Run and Candidate APIs. Run and # Candidate objects are persisted in etcd; incompatible schema changes may make - # them unreadable or block future CRD upgrades. The operator does not reconcile - # v1beta2 requests. + # them unreadable or block future CRD upgrades. When true, the operator also + # reconciles v1beta2 Requests into Runs; the controller that executes a Run is + # not part of the operator yet. enabled: false # Environment variables to pass to operator Deployment. diff --git a/deploy/operator/cmd/main.go b/deploy/operator/cmd/main.go index 3c762841c7a2..27fe6de2b945 100644 --- a/deploy/operator/cmd/main.go +++ b/deploy/operator/cmd/main.go @@ -164,6 +164,7 @@ func main() { var operatorImagePullPolicy string var dgdrDefaultImage string var dgdrAISimulateDefaultImage string + var dgdrV1beta2Enabled bool flag.StringVar(&configFile, "config", "", "Path to operator configuration file (required)") flag.StringVar(&operatorVersion, "operator-version", "unknown", "Version of the operator (used in lease holder identity)") @@ -180,6 +181,8 @@ func main() { flag.StringVar(&dgdrAISimulateDefaultImage, "dgdr-aisimulate-default-image", "", "AISimulate runner image, put into v1beta2 DGDR spec.aisimulate.image when unset; "+ "empty derives dynamo-planner:") + flag.BoolVar(&dgdrV1beta2Enabled, "dgdr-v1beta2-enabled", false, + "Reconcile native v1beta2 DGDRs; requires the v1beta2 Request and Run CRDs to be installed") opts := zap.Options{ Development: true, } @@ -506,6 +509,7 @@ func main() { mgr, operatorCfg, runtimeConfig, dockerSecretRetriever, sshKeyManager, operatorImage, pullPolicy, + dgdrV1beta2Enabled, ); err != nil { setupLog.Error(err, "failed to register controllers") os.Exit(1) @@ -555,6 +559,7 @@ func registerControllers( sshKeyManager *secret.SSHKeyManager, operatorImage string, operatorPullPolicy corev1.PullPolicy, + dgdrV1beta2Enabled bool, ) error { if operatorCfg.Namespace.Restricted == "" { // The cluster-wide manager cache covers all namespaces. ExcludedNamespaces only filters @@ -619,6 +624,11 @@ func registerControllers( }); err != nil { return err } + if dgdrV1beta2Enabled { + if err := controller.SetupDynamoGraphDeploymentRequestV1beta2(mgr, setupOptions); err != nil { + return err + } + } if err := controller.SetupDynamoModel(mgr, controller.DynamoModelSetupOptions{ SetupOptions: setupOptions, }); err != nil { diff --git a/deploy/operator/internal/controller/dynamographdeploymentrequest_v1beta2_controller.go b/deploy/operator/internal/controller/dynamographdeploymentrequest_v1beta2_controller.go new file mode 100644 index 000000000000..a9ffc2d1597d --- /dev/null +++ b/deploy/operator/internal/controller/dynamographdeploymentrequest_v1beta2_controller.go @@ -0,0 +1,332 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package controller + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "sort" + "strconv" + + configv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/config/v1alpha1" + nvidiacomv1beta2 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta2" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/controller/aisimulate" + commonController "github.com/ai-dynamo/dynamo/deploy/operator/internal/controller_common" + corev1 "k8s.io/api/core/v1" + apiequality "k8s.io/apimachinery/pkg/api/equality" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/log" +) + +const ( + // dgdrV1beta2RequestLabel names the Request that owns a Run. + dgdrV1beta2RequestLabel = "nvidia.com/dgdr-request" + // dgdrV1beta2GenerationAnnotation records the Request generation a Run was created for. + dgdrV1beta2GenerationAnnotation = "nvidia.com/dgdr-request-generation" + + // dgdrV1beta2RetainedRuns is how many terminal Runs a Request keeps. Older ones are deleted. + dgdrV1beta2RetainedRuns = 5 + + // DGDRReconciledConditionV1beta2 reports whether the controller accepted the current generation. + DGDRReconciledConditionV1beta2 = "Reconciled" + + dgdrReasonRunCreated = "RunCreated" + dgdrReasonInvalidSpec = "InvalidSpec" + dgdrReasonWaitingForActiveRun = "WaitingForActiveRun" + + dgdrRunPhaseSucceeded = "Succeeded" + dgdrRunPhaseFailed = "Failed" +) + +// DynamoGraphDeploymentRequestV1beta2Reconciler turns an accepted native v1beta2 Request into an +// immutable Run. It never reads the cluster for hardware: admission has already supplied it, and a +// Request that still lacks it is reported instead of searched for. The Run controller executes the Run. +type DynamoGraphDeploymentRequestV1beta2Reconciler struct { + client.Client + Config *configv1alpha1.OperatorConfiguration + RuntimeConfig *commonController.RuntimeConfig +} + +// +kubebuilder:rbac:groups=nvidia.com,resources=dynamographdeploymentrequests,verbs=get;list;watch +// +kubebuilder:rbac:groups=nvidia.com,resources=dynamographdeploymentrequests/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=nvidia.com,resources=dynamographdeploymentruns,verbs=get;list;watch;create;delete + +// Reconcile creates the Run for the current generation of a native v1beta2 Request once no other Run +// is active, and records the active and retained Runs in the Request status. +func (r *DynamoGraphDeploymentRequestV1beta2Reconciler) Reconcile( + ctx context.Context, + req ctrl.Request, +) (ctrl.Result, error) { + logger := log.FromContext(ctx) + + request := &nvidiacomv1beta2.DynamoGraphDeploymentRequest{} + if err := r.Get(ctx, req.NamespacedName, request); err != nil { + return ctrl.Result{}, client.IgnoreNotFound(err) + } + // A native v1beta1 Request is a read-only carrier here; the v1beta1 reconciler owns it. + if request.DeletionTimestamp != nil || request.Spec.V1Beta1 != nil { + return ctrl.Result{}, nil + } + original := request.DeepCopy() + + runs, err := r.ownedRuns(ctx, request) + if err != nil { + return ctrl.Result{}, err + } + + wantName := dgdrV1beta2RunName(request) + var current *nvidiacomv1beta2.DynamoGraphDeploymentRun + for i := range runs { + if runs[i].Name == wantName { + current = &runs[i] + } + } + + accepted := current != nil + condition := dgdrV1beta2Condition(request, metav1.ConditionTrue, dgdrReasonRunCreated, + fmt.Sprintf("Run %q exists for generation %d", wantName, request.Generation)) + if current == nil { + if active := dgdrV1beta2NewestActive(runs); active != nil { + condition = dgdrV1beta2Condition(request, metav1.ConditionFalse, dgdrReasonWaitingForActiveRun, + fmt.Sprintf("Run %q is still active; the Run for generation %d starts when it finishes", + active.Name, request.Generation)) + } else if run, buildErr := r.buildRun(request); buildErr != nil { + logger.Info("Request cannot be turned into a Run", "reason", buildErr.Error()) + condition = dgdrV1beta2Condition(request, metav1.ConditionFalse, dgdrReasonInvalidSpec, buildErr.Error()) + } else { + if err := r.Create(ctx, run); err != nil && !apierrors.IsAlreadyExists(err) { + return ctrl.Result{}, fmt.Errorf("create Run %q: %w", wantName, err) + } + runs = append(runs, *run) + accepted = true + } + } + + active, terminal := dgdrV1beta2SplitRuns(runs) + keep := min(len(terminal), dgdrV1beta2RetainedRuns) + for i := keep; i < len(terminal); i++ { + if err := r.Delete(ctx, &terminal[i]); err != nil && !apierrors.IsNotFound(err) { + return ctrl.Result{}, fmt.Errorf("delete retired Run %q: %w", terminal[i].Name, err) + } + } + + if accepted { + request.Status.ObservedGeneration = request.Generation + } + request.Status.ActiveRunRef = nil + if active != nil { + request.Status.ActiveRunRef = &corev1.LocalObjectReference{Name: active.Name} + } + request.Status.RecentRunRefs = nil + for i := 0; i < keep; i++ { + request.Status.RecentRunRefs = append(request.Status.RecentRunRefs, corev1.LocalObjectReference{Name: terminal[i].Name}) + } + meta.SetStatusCondition(&request.Status.Conditions, condition) + + if !apiequality.Semantic.DeepEqual(original.Status, request.Status) { + if err := r.Status().Patch(ctx, request, client.MergeFrom(original)); err != nil { + return ctrl.Result{}, fmt.Errorf("update Request status: %w", err) + } + } + return ctrl.Result{}, nil +} + +// buildRun resolves the optional inputs of the Request into an immutable Run spec. The effective +// AISimulate input is frozen into aisimulate.parameters, which is authoritative, so lowering the Run +// again yields the same bytes. +func (r *DynamoGraphDeploymentRequestV1beta2Reconciler) buildRun( + request *nvidiacomv1beta2.DynamoGraphDeploymentRequest, +) (*nvidiacomv1beta2.DynamoGraphDeploymentRun, error) { + spec := &request.Spec + if spec.AISimulate == nil || spec.AISimulate.Image == "" { + return nil, errors.New("spec.aisimulate.image is required") + } + version, err := aisimulate.ResolveVersion(spec.AISimulate.Image, spec.AISimulate.VersionOverride) + if err != nil { + return nil, err + } + input, err := aisimulate.Lower(spec) + if err != nil { + return nil, err + } + if !dgdrV1beta2HasEffectiveValue(input, aisimulate.SearchSpaceHardwareSKU) || + !dgdrV1beta2HasEffectiveValue(input, aisimulate.SearchSpaceGPUBudget) { + return nil, errors.New("the effective AISimulate input names no GPU SKU or budget; set spec.hardware.gpu") + } + raw, err := json.Marshal(input) + if err != nil { + return nil, fmt.Errorf("encode the effective AISimulate input: %w", err) + } + + frozen := spec.NativeSearchInput.DeepCopy() + frozen.AISimulate = &nvidiacomv1beta2.AISimulateSpec{ + Image: spec.AISimulate.Image, + VersionOverride: version, + Parameters: &runtime.RawExtension{Raw: raw}, + } + run := &nvidiacomv1beta2.DynamoGraphDeploymentRun{ + ObjectMeta: metav1.ObjectMeta{ + Name: dgdrV1beta2RunName(request), + Namespace: request.Namespace, + Labels: map[string]string{dgdrV1beta2RequestLabel: request.Name}, + Annotations: map[string]string{dgdrV1beta2GenerationAnnotation: strconv.FormatInt(request.Generation, 10)}, + }, + Spec: nvidiacomv1beta2.DynamoGraphDeploymentRunSpec{NativeSearchInput: *frozen}, + } + if err := controllerutil.SetControllerReference(request, run, r.Scheme()); err != nil { + return nil, fmt.Errorf("set Run owner: %w", err) + } + return run, nil +} + +// ownedRuns returns the Runs this Request controls. +func (r *DynamoGraphDeploymentRequestV1beta2Reconciler) ownedRuns( + ctx context.Context, + request *nvidiacomv1beta2.DynamoGraphDeploymentRequest, +) ([]nvidiacomv1beta2.DynamoGraphDeploymentRun, error) { + list := &nvidiacomv1beta2.DynamoGraphDeploymentRunList{} + if err := r.List(ctx, list, client.InNamespace(request.Namespace), + client.MatchingLabels{dgdrV1beta2RequestLabel: request.Name}); err != nil { + return nil, fmt.Errorf("list Runs: %w", err) + } + owned := make([]nvidiacomv1beta2.DynamoGraphDeploymentRun, 0, len(list.Items)) + for i := range list.Items { + if metav1.IsControlledBy(&list.Items[i], request) { + owned = append(owned, list.Items[i]) + } + } + return owned, nil +} + +// SetupWithManager registers the controller. It watches the Runs it owns, so a finished Run wakes +// the Request that was waiting for it. +func (r *DynamoGraphDeploymentRequestV1beta2Reconciler) SetupWithManager(mgr ctrl.Manager) error { + return ctrl.NewControllerManagedBy(mgr). + Named("dynamographdeploymentrequest-v1beta2"). + For(&nvidiacomv1beta2.DynamoGraphDeploymentRequest{}). + Owns(&nvidiacomv1beta2.DynamoGraphDeploymentRun{}). + WithEventFilter(commonController.EphemeralDeploymentEventFilter(r.Config, r.RuntimeConfig)). + Complete(r) +} + +// dgdrV1beta2RunName names the Run of one Request generation. A metadata-only update keeps the +// generation, so it never creates a second Run. +func dgdrV1beta2RunName(request *nvidiacomv1beta2.DynamoGraphDeploymentRequest) string { + return fmt.Sprintf("%s-%d", request.Name, request.Generation) +} + +func dgdrV1beta2Condition( + request *nvidiacomv1beta2.DynamoGraphDeploymentRequest, + status metav1.ConditionStatus, + reason, message string, +) metav1.Condition { + return metav1.Condition{ + Type: DGDRReconciledConditionV1beta2, + Status: status, + Reason: reason, + Message: message, + ObservedGeneration: request.Generation, + } +} + +func dgdrV1beta2IsTerminal(run *nvidiacomv1beta2.DynamoGraphDeploymentRun) bool { + phase := string(run.Status.Phase) + return phase == dgdrRunPhaseSucceeded || phase == dgdrRunPhaseFailed +} + +// dgdrV1beta2Newer orders Runs newest first: by Request generation, then creation time, then name. +func dgdrV1beta2Newer(a, b *nvidiacomv1beta2.DynamoGraphDeploymentRun) bool { + ga, gb := dgdrV1beta2RunGeneration(a), dgdrV1beta2RunGeneration(b) + if ga != gb { + return ga > gb + } + if !a.CreationTimestamp.Equal(&b.CreationTimestamp) { + return b.CreationTimestamp.Before(&a.CreationTimestamp) + } + return a.Name > b.Name +} + +func dgdrV1beta2RunGeneration(run *nvidiacomv1beta2.DynamoGraphDeploymentRun) int64 { + generation, _ := strconv.ParseInt(run.Annotations[dgdrV1beta2GenerationAnnotation], 10, 64) + return generation +} + +// dgdrV1beta2SplitRuns returns the newest non-terminal Run, if any, and the terminal Runs newest first. +func dgdrV1beta2SplitRuns( + runs []nvidiacomv1beta2.DynamoGraphDeploymentRun, +) (*nvidiacomv1beta2.DynamoGraphDeploymentRun, []nvidiacomv1beta2.DynamoGraphDeploymentRun) { + sorted := append([]nvidiacomv1beta2.DynamoGraphDeploymentRun(nil), runs...) + sort.SliceStable(sorted, func(i, j int) bool { return dgdrV1beta2Newer(&sorted[i], &sorted[j]) }) + var active *nvidiacomv1beta2.DynamoGraphDeploymentRun + var terminal []nvidiacomv1beta2.DynamoGraphDeploymentRun + for i := range sorted { + switch { + case !dgdrV1beta2IsTerminal(&sorted[i]): + if active == nil { + active = &sorted[i] + } + default: + terminal = append(terminal, sorted[i]) + } + } + return active, terminal +} + +func dgdrV1beta2NewestActive( + runs []nvidiacomv1beta2.DynamoGraphDeploymentRun, +) *nvidiacomv1beta2.DynamoGraphDeploymentRun { + active, _ := dgdrV1beta2SplitRuns(runs) + return active +} + +// dgdrV1beta2HasEffectiveValue reports whether input holds a non-null, non-empty value at path. +func dgdrV1beta2HasEffectiveValue(input map[string]any, path []string) bool { + value, found, err := unstructured.NestedFieldNoCopy(input, path...) + if err != nil || !found || value == nil { + return false + } + text, isString := value.(string) + return !isString || text != "" +} + +// requireDGDRV1beta2API fails when the cluster does not serve the v1beta2 Request and Run APIs. +// Without them the controller's caches never sync and the manager would hang at startup. +func requireDGDRV1beta2API(mgr ctrl.Manager) error { + group, version := nvidiacomv1beta2.GroupVersion.Group, nvidiacomv1beta2.GroupVersion.Version + for _, kind := range []string{"DynamoGraphDeploymentRequest", "DynamoGraphDeploymentRun"} { + _, err := mgr.GetRESTMapper().RESTMapping(schema.GroupKind{Group: group, Kind: kind}, version) + if meta.IsNoMatchError(err) { + return fmt.Errorf("--dgdr-v1beta2-enabled is set but the cluster does not serve %s/%s %s; "+ + "install the v1beta2 CRDs (dgdrV1Beta2.enabled)", group, version, kind) + } + if err != nil { + return fmt.Errorf("check that %s/%s %s is served: %w", group, version, kind, err) + } + } + return nil +} diff --git a/deploy/operator/internal/controller/dynamographdeploymentrequest_v1beta2_controller_envtest_test.go b/deploy/operator/internal/controller/dynamographdeploymentrequest_v1beta2_controller_envtest_test.go new file mode 100644 index 000000000000..8db3a12f80b1 --- /dev/null +++ b/deploy/operator/internal/controller/dynamographdeploymentrequest_v1beta2_controller_envtest_test.go @@ -0,0 +1,122 @@ +//go:build !clustertest + +/* + * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package controller + +import ( + "context" + "strconv" + + nvidiacomv1beta2 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta2" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +var _ = Describe("DynamoGraphDeploymentRequest v1beta2 controller", func() { + var reconciler *DynamoGraphDeploymentRequestV1beta2Reconciler + + BeforeEach(func() { + reconciler = &DynamoGraphDeploymentRequestV1beta2Reconciler{Client: k8sClient} + }) + + reconcile := func(ctx context.Context, key types.NamespacedName) { + _, err := reconciler.Reconcile(ctx, ctrl.Request{NamespacedName: key}) + Expect(err).NotTo(HaveOccurred()) + } + + It("Should create one Run per generation through the API server", func() { + ctx := context.Background() + dgdr := &nvidiacomv1beta2.DynamoGraphDeploymentRequest{ + ObjectMeta: metav1.ObjectMeta{Name: "v1beta2-run-per-generation", Namespace: envtestNamespace}, + Spec: nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec{ + NativeSearchInput: nvidiacomv1beta2.NativeSearchInput{ + Model: &nvidiacomv1beta2.ModelSpec{Name: "Qwen/Qwen3-32B"}, + }, + }, + } + + By("Creating a native v1beta2 request; admission supplies the image and the hardware") + Expect(k8sClient.Create(ctx, dgdr)).Should(Succeed()) + defer func() { _ = k8sClient.Delete(ctx, dgdr) }() + key := client.ObjectKeyFromObject(dgdr) + + By("Reconciling creates the Run of generation 1") + reconcile(ctx, key) + live := &nvidiacomv1beta2.DynamoGraphDeploymentRequest{} + Expect(k8sClient.Get(ctx, key, live)).Should(Succeed()) + firstName := dgdr.Name + "-" + strconv.FormatInt(live.Generation, 10) + first := &nvidiacomv1beta2.DynamoGraphDeploymentRun{} + Expect(k8sClient.Get(ctx, types.NamespacedName{Namespace: envtestNamespace, Name: firstName}, first)).Should(Succeed()) + Expect(metav1.IsControlledBy(first, live)).To(BeTrue()) + Expect(first.Spec.Hardware).NotTo(BeNil()) + Expect(first.Spec.AISimulate.Image).To(Equal(controllerAISimulateImage)) + Expect(first.Spec.AISimulate.VersionOverride).To(Equal("1.2.0")) + Expect(first.Spec.AISimulate.Parameters).NotTo(BeNil()) + + By("Reading the status the controller wrote through the status subresource") + Expect(live.Status.ObservedGeneration).To(Equal(live.Generation)) + Expect(live.Status.ActiveRunRef).NotTo(BeNil()) + Expect(live.Status.ActiveRunRef.Name).To(Equal(firstName)) + condition := meta.FindStatusCondition(live.Status.Conditions, DGDRReconciledConditionV1beta2) + Expect(condition).NotTo(BeNil()) + Expect(condition.Status).To(Equal(metav1.ConditionTrue)) + Expect(live.Status.V1Beta1).To(BeNil()) + + By("Reconciling again creates nothing") + reconcile(ctx, key) + runs := &nvidiacomv1beta2.DynamoGraphDeploymentRunList{} + Expect(k8sClient.List(ctx, runs, client.InNamespace(envtestNamespace))).Should(Succeed()) + Expect(runs.Items).To(HaveLen(1)) + + By("Changing the spec while the Run is active starts nothing and says why") + Expect(k8sClient.Get(ctx, key, live)).Should(Succeed()) + live.Spec.Model.Name = "Qwen/Qwen3-30B-A3B" + Expect(k8sClient.Update(ctx, live)).Should(Succeed()) + reconcile(ctx, key) + Expect(k8sClient.List(ctx, runs, client.InNamespace(envtestNamespace))).Should(Succeed()) + Expect(runs.Items).To(HaveLen(1)) + Expect(k8sClient.Get(ctx, key, live)).Should(Succeed()) + Expect(live.Generation).To(BeNumerically(">", 1)) + condition = meta.FindStatusCondition(live.Status.Conditions, DGDRReconciledConditionV1beta2) + Expect(condition).NotTo(BeNil()) + Expect(condition.Reason).To(Equal(dgdrReasonWaitingForActiveRun)) + + By("Finishing the first Run lets the next generation start") + Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(first), first)).Should(Succeed()) + first.Status.Phase = nvidiacomv1beta2.DGDRRunPhase(dgdrRunPhaseSucceeded) + Expect(k8sClient.Status().Update(ctx, first)).Should(Succeed()) + reconcile(ctx, key) + + Expect(k8sClient.Get(ctx, key, live)).Should(Succeed()) + secondName := dgdr.Name + "-" + strconv.FormatInt(live.Generation, 10) + Expect(live.Status.ObservedGeneration).To(Equal(live.Generation)) + Expect(live.Status.ActiveRunRef).NotTo(BeNil()) + Expect(live.Status.ActiveRunRef.Name).To(Equal(secondName)) + Expect(live.Status.RecentRunRefs).To(HaveLen(1)) + Expect(live.Status.RecentRunRefs[0].Name).To(Equal(firstName)) + second := &nvidiacomv1beta2.DynamoGraphDeploymentRun{} + Expect(k8sClient.Get(ctx, types.NamespacedName{Namespace: envtestNamespace, Name: secondName}, second)).Should(Succeed()) + Expect(second.Spec.Model.Name).To(Equal("Qwen/Qwen3-30B-A3B")) + }) +}) diff --git a/deploy/operator/internal/controller/dynamographdeploymentrequest_v1beta2_controller_test.go b/deploy/operator/internal/controller/dynamographdeploymentrequest_v1beta2_controller_test.go new file mode 100644 index 000000000000..821fdd19f2df --- /dev/null +++ b/deploy/operator/internal/controller/dynamographdeploymentrequest_v1beta2_controller_test.go @@ -0,0 +1,298 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package controller + +import ( + "encoding/json" + "fmt" + "testing" + + nvidiacomv1beta2 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta2" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +type dgdrV1beta2Fixture struct { + t *testing.T + client client.Client + reconciler *DynamoGraphDeploymentRequestV1beta2Reconciler +} + +func newDGDRV1beta2Fixture(t *testing.T, objects ...client.Object) *dgdrV1beta2Fixture { + t.Helper() + scheme := runtime.NewScheme() + if err := nvidiacomv1beta2.AddToScheme(scheme); err != nil { + t.Fatalf("add v1beta2 to scheme: %v", err) + } + c := fake.NewClientBuilder(). + WithScheme(scheme). + WithObjects(objects...). + WithStatusSubresource(&nvidiacomv1beta2.DynamoGraphDeploymentRequest{}, &nvidiacomv1beta2.DynamoGraphDeploymentRun{}). + Build() + return &dgdrV1beta2Fixture{t: t, client: c, reconciler: &DynamoGraphDeploymentRequestV1beta2Reconciler{Client: c}} +} + +func (f *dgdrV1beta2Fixture) reconcile() { + f.t.Helper() + if _, err := f.reconciler.Reconcile(f.t.Context(), ctrl.Request{ + NamespacedName: types.NamespacedName{Namespace: "default", Name: "search"}, + }); err != nil { + f.t.Fatalf("reconcile: %v", err) + } +} + +func (f *dgdrV1beta2Fixture) request() *nvidiacomv1beta2.DynamoGraphDeploymentRequest { + f.t.Helper() + request := &nvidiacomv1beta2.DynamoGraphDeploymentRequest{} + if err := f.client.Get(f.t.Context(), types.NamespacedName{Namespace: "default", Name: "search"}, request); err != nil { + f.t.Fatalf("get request: %v", err) + } + return request +} + +func (f *dgdrV1beta2Fixture) runNames() []string { + f.t.Helper() + list := &nvidiacomv1beta2.DynamoGraphDeploymentRunList{} + if err := f.client.List(f.t.Context(), list, client.InNamespace("default")); err != nil { + f.t.Fatalf("list runs: %v", err) + } + names := make([]string, 0, len(list.Items)) + for _, run := range list.Items { + names = append(names, run.Name) + } + return names +} + +func (f *dgdrV1beta2Fixture) setPhase(name, phase string) { + f.t.Helper() + run := &nvidiacomv1beta2.DynamoGraphDeploymentRun{} + if err := f.client.Get(f.t.Context(), types.NamespacedName{Namespace: "default", Name: name}, run); err != nil { + f.t.Fatalf("get run %s: %v", name, err) + } + run.Status.Phase = nvidiacomv1beta2.DGDRRunPhase(phase) + if err := f.client.Status().Update(f.t.Context(), run); err != nil { + f.t.Fatalf("set phase of run %s: %v", name, err) + } +} + +func (f *dgdrV1beta2Fixture) bumpGeneration(generation int64) { + f.t.Helper() + request := f.request() + request.Generation = generation + if err := f.client.Update(f.t.Context(), request); err != nil { + f.t.Fatalf("bump generation: %v", err) + } +} + +func dgdrV1beta2Request(withHardware bool) *nvidiacomv1beta2.DynamoGraphDeploymentRequest { + request := &nvidiacomv1beta2.DynamoGraphDeploymentRequest{ + ObjectMeta: metav1.ObjectMeta{Name: "search", Namespace: "default", UID: "request-uid", Generation: 1}, + } + request.Spec.Model = &nvidiacomv1beta2.ModelSpec{Name: "Qwen/Qwen3-0.6B"} + request.Spec.AISimulate = &nvidiacomv1beta2.AISimulateSpec{ + Image: "example.com/aisimulate:latest", + VersionOverride: "0.13.0", + } + if withHardware { + sku, budget := "h100_sxm", int32(4) + request.Spec.Hardware = &nvidiacomv1beta2.HardwareSpec{ + GPU: &nvidiacomv1beta2.GPUHardwareSpec{SKU: &sku, Budget: &budget}, + } + } + return request +} + +func dgdrV1beta2ReconciledCondition(t *testing.T, request *nvidiacomv1beta2.DynamoGraphDeploymentRequest) metav1.Condition { + t.Helper() + condition := meta.FindStatusCondition(request.Status.Conditions, DGDRReconciledConditionV1beta2) + if condition == nil { + t.Fatal("no Reconciled condition") + } + return *condition +} + +func TestV1beta2RequestControllerCreatesTheRunOfTheCurrentGeneration(t *testing.T) { + f := newDGDRV1beta2Fixture(t, dgdrV1beta2Request(true)) + f.reconcile() + + t.Log("One Run exists, owned by the Request, with the resolved and frozen input") + run := &nvidiacomv1beta2.DynamoGraphDeploymentRun{} + if err := f.client.Get(t.Context(), types.NamespacedName{Namespace: "default", Name: "search-1"}, run); err != nil { + t.Fatalf("get run: %v", err) + } + if !metav1.IsControlledBy(run, f.request()) { + t.Fatal("run is not controlled by the request") + } + if got := run.Spec.AISimulate.VersionOverride; got != "0.13.0" { + t.Fatalf("run versionOverride = %q, want the resolved 0.13.0", got) + } + var parameters map[string]any + if err := json.Unmarshal(run.Spec.AISimulate.Parameters.Raw, ¶meters); err != nil { + t.Fatalf("decode frozen parameters: %v", err) + } + searchSpace, _ := parameters["search_space"].(map[string]any) + if searchSpace["hardware_sku"] != "h100_sxm" || searchSpace["gpu_budget"] != float64(4) { + t.Fatalf("frozen search_space = %v, want the hardware of the request", searchSpace) + } + + t.Log("The status records the accepted generation and the active Run") + request := f.request() + if request.Status.ObservedGeneration != 1 { + t.Fatalf("observedGeneration = %d, want 1", request.Status.ObservedGeneration) + } + if request.Status.ActiveRunRef == nil || request.Status.ActiveRunRef.Name != "search-1" { + t.Fatalf("activeRunRef = %v, want search-1", request.Status.ActiveRunRef) + } + if condition := dgdrV1beta2ReconciledCondition(t, request); condition.Status != metav1.ConditionTrue { + t.Fatalf("Reconciled = %v, want True", condition) + } +} + +func TestV1beta2RequestControllerIsIdempotent(t *testing.T) { + f := newDGDRV1beta2Fixture(t, dgdrV1beta2Request(true)) + f.reconcile() + f.reconcile() + f.reconcile() + if names := f.runNames(); len(names) != 1 || names[0] != "search-1" { + t.Fatalf("runs = %v, want only search-1", names) + } +} + +func TestV1beta2RequestControllerWaitsForTheActiveRunBeforeStartingTheNext(t *testing.T) { + f := newDGDRV1beta2Fixture(t, dgdrV1beta2Request(true)) + f.reconcile() + + t.Log("A spec change while search-1 runs creates nothing and says why") + f.bumpGeneration(2) + f.reconcile() + if names := f.runNames(); len(names) != 1 { + t.Fatalf("runs = %v, want only the active run", names) + } + request := f.request() + if request.Status.ObservedGeneration != 1 { + t.Fatalf("observedGeneration = %d, want 1 until generation 2 is accepted", request.Status.ObservedGeneration) + } + if condition := dgdrV1beta2ReconciledCondition(t, request); condition.Reason != dgdrReasonWaitingForActiveRun { + t.Fatalf("condition = %v, want reason %s", condition, dgdrReasonWaitingForActiveRun) + } + + t.Log("Once search-1 is terminal, search-2 is created and search-1 moves to the history") + f.setPhase("search-1", dgdrRunPhaseSucceeded) + f.reconcile() + request = f.request() + if request.Status.ObservedGeneration != 2 { + t.Fatalf("observedGeneration = %d, want 2", request.Status.ObservedGeneration) + } + if request.Status.ActiveRunRef == nil || request.Status.ActiveRunRef.Name != "search-2" { + t.Fatalf("activeRunRef = %v, want search-2", request.Status.ActiveRunRef) + } + if len(request.Status.RecentRunRefs) != 1 || request.Status.RecentRunRefs[0].Name != "search-1" { + t.Fatalf("recentRunRefs = %v, want [search-1]", request.Status.RecentRunRefs) + } +} + +func TestV1beta2RequestControllerCreatesNoRunWithoutHardware(t *testing.T) { + f := newDGDRV1beta2Fixture(t, dgdrV1beta2Request(false)) + f.reconcile() + if names := f.runNames(); len(names) != 0 { + t.Fatalf("runs = %v, want none", names) + } + request := f.request() + condition := dgdrV1beta2ReconciledCondition(t, request) + if condition.Status != metav1.ConditionFalse || condition.Reason != dgdrReasonInvalidSpec { + t.Fatalf("condition = %v, want False/%s", condition, dgdrReasonInvalidSpec) + } + if request.Status.ObservedGeneration != 0 { + t.Fatalf("observedGeneration = %d, want 0 for a rejected generation", request.Status.ObservedGeneration) + } +} + +func TestV1beta2RequestControllerLeavesForeignCarriersAlone(t *testing.T) { + request := dgdrV1beta2Request(true) + carrier := request.DeepCopy() + carrier.Spec.V1Beta1 = &apiextensionsJSONForTest + f := newDGDRV1beta2Fixture(t, carrier) + f.reconcile() + if names := f.runNames(); len(names) != 0 { + t.Fatalf("runs = %v, want none for a v1beta1 carrier", names) + } + if got := f.request().Status.ObservedGeneration; got != 0 { + t.Fatalf("observedGeneration = %d, want the carrier's status untouched", got) + } +} + +func TestV1beta2RequestControllerRetainsFiveTerminalRuns(t *testing.T) { + request := dgdrV1beta2Request(true) + request.Generation = 8 + objects := []client.Object{request} + for generation := 1; generation <= 7; generation++ { + objects = append(objects, dgdrV1beta2TerminalRun(request, generation)) + } + f := newDGDRV1beta2Fixture(t, objects...) + f.reconcile() + + t.Log("The two oldest terminal runs are deleted and the newest five are listed newest first") + for _, name := range []string{"search-1", "search-2"} { + err := f.client.Get(t.Context(), types.NamespacedName{Namespace: "default", Name: name}, &nvidiacomv1beta2.DynamoGraphDeploymentRun{}) + if !apierrors.IsNotFound(err) { + t.Fatalf("run %s: err = %v, want it deleted", name, err) + } + } + refs := f.request().Status.RecentRunRefs + want := []string{"search-7", "search-6", "search-5", "search-4", "search-3"} + if len(refs) != len(want) { + t.Fatalf("recentRunRefs = %v, want %v", refs, want) + } + for i := range want { + if refs[i].Name != want[i] { + t.Fatalf("recentRunRefs = %v, want %v", refs, want) + } + } +} + +func dgdrV1beta2TerminalRun(request *nvidiacomv1beta2.DynamoGraphDeploymentRequest, generation int) *nvidiacomv1beta2.DynamoGraphDeploymentRun { + run := &nvidiacomv1beta2.DynamoGraphDeploymentRun{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("search-%d", generation), + Namespace: "default", + Labels: map[string]string{dgdrV1beta2RequestLabel: request.Name}, + Annotations: map[string]string{dgdrV1beta2GenerationAnnotation: fmt.Sprint(generation)}, + OwnerReferences: []metav1.OwnerReference{{ + APIVersion: nvidiacomv1beta2.GroupVersion.String(), + Kind: "DynamoGraphDeploymentRequest", + Name: request.Name, + UID: request.UID, + Controller: boolPointerForTest(true), + }}, + }, + } + run.Status.Phase = nvidiacomv1beta2.DGDRRunPhase(dgdrRunPhaseSucceeded) + return run +} + +func boolPointerForTest(value bool) *bool { return &value } + +// apiextensionsJSONForTest is a non-empty v1beta1 carrier payload. +var apiextensionsJSONForTest = apiextensionsv1.JSON{Raw: []byte(`{"model":"carrier"}`)} diff --git a/deploy/operator/internal/controller/setup.go b/deploy/operator/internal/controller/setup.go index a8704ddbca64..42a081839153 100644 --- a/deploy/operator/internal/controller/setup.go +++ b/deploy/operator/internal/controller/setup.go @@ -133,6 +133,22 @@ func SetupDynamoGraphDeploymentRequest(mgr ctrl.Manager, opts DynamoGraphDeploym return nil } +// SetupDynamoGraphDeploymentRequestV1beta2 registers the controller that turns native v1beta2 Requests +// into Runs. It fails at startup, with a clear message, when the cluster does not serve the v1beta2 APIs. +func SetupDynamoGraphDeploymentRequestV1beta2(mgr ctrl.Manager, opts SetupOptions) error { + if err := requireDGDRV1beta2API(mgr); err != nil { + return err + } + if err := (&DynamoGraphDeploymentRequestV1beta2Reconciler{ + Client: mgr.GetClient(), + Config: opts.Config, + RuntimeConfig: opts.RuntimeConfig, + }).SetupWithManager(mgr); err != nil { + return fmt.Errorf("unable to create DynamoGraphDeploymentRequest v1beta2 controller: %w", err) + } + return nil +} + func SetupDynamoModel(mgr ctrl.Manager, opts DynamoModelSetupOptions) error { if err := (&DynamoModelReconciler{ Client: mgr.GetClient(), From d23dfbd5775df75cd31a1325bb97af8fa528deb4 Mon Sep 17 00:00:00 2001 From: devivasudevan <49675305+devivasudevan@users.noreply.github.com> Date: Thu, 22 Oct 2026 13:14:12 +0200 Subject: [PATCH 25/40] fix(operator): harden v1beta2 Request controller Run ownership and naming Label Runs with the Request UID (label values are limited to 63 characters), refuse to adopt a Run of the same name owned by another object, and report an over-long derived Run name as InvalidSpec. Signed-off-by: devivasudevan <49675305+devivasudevan@users.noreply.github.com> --- ...aphdeploymentrequest_v1beta2_controller.go | 48 ++++++++++--- ...ploymentrequest_v1beta2_controller_test.go | 67 ++++++++++++++++++- 2 files changed, 106 insertions(+), 9 deletions(-) diff --git a/deploy/operator/internal/controller/dynamographdeploymentrequest_v1beta2_controller.go b/deploy/operator/internal/controller/dynamographdeploymentrequest_v1beta2_controller.go index a9ffc2d1597d..3cab7bb4075f 100644 --- a/deploy/operator/internal/controller/dynamographdeploymentrequest_v1beta2_controller.go +++ b/deploy/operator/internal/controller/dynamographdeploymentrequest_v1beta2_controller.go @@ -24,6 +24,7 @@ import ( "fmt" "sort" "strconv" + "time" configv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/config/v1alpha1" nvidiacomv1beta2 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta2" @@ -44,8 +45,9 @@ import ( ) const ( - // dgdrV1beta2RequestLabel names the Request that owns a Run. - dgdrV1beta2RequestLabel = "nvidia.com/dgdr-request" + // dgdrV1beta2RequestLabel holds the UID of the Request that owns a Run. A label value is limited to + // 63 characters, which a Request name is not, so the UID is used. + dgdrV1beta2RequestLabel = "nvidia.com/dgdr-request-uid" // dgdrV1beta2GenerationAnnotation records the Request generation a Run was created for. dgdrV1beta2GenerationAnnotation = "nvidia.com/dgdr-request-generation" @@ -58,6 +60,13 @@ const ( dgdrReasonRunCreated = "RunCreated" dgdrReasonInvalidSpec = "InvalidSpec" dgdrReasonWaitingForActiveRun = "WaitingForActiveRun" + dgdrReasonRunNameConflict = "RunNameConflict" + + // dgdrV1beta2MaxRunNameLength is the longest object name Kubernetes accepts. + dgdrV1beta2MaxRunNameLength = 253 + + // dgdrV1beta2ConflictRetry is how long to wait before looking again at a Run name held by another owner. + dgdrV1beta2ConflictRetry = 5 * time.Second dgdrRunPhaseSucceeded = "Succeeded" dgdrRunPhaseFailed = "Failed" @@ -93,6 +102,7 @@ func (r *DynamoGraphDeploymentRequestV1beta2Reconciler) Reconcile( return ctrl.Result{}, nil } original := request.DeepCopy() + result := ctrl.Result{} runs, err := r.ownedRuns(ctx, request) if err != nil { @@ -119,11 +129,29 @@ func (r *DynamoGraphDeploymentRequestV1beta2Reconciler) Reconcile( logger.Info("Request cannot be turned into a Run", "reason", buildErr.Error()) condition = dgdrV1beta2Condition(request, metav1.ConditionFalse, dgdrReasonInvalidSpec, buildErr.Error()) } else { - if err := r.Create(ctx, run); err != nil && !apierrors.IsAlreadyExists(err) { + err := r.Create(ctx, run) + switch { + case err == nil: + runs = append(runs, *run) + accepted = true + case apierrors.IsAlreadyExists(err): + // A stale cache, or a Run left by an earlier Request of the same name that is still being + // garbage collected. Only a Run this Request controls counts as ours. + existing := &nvidiacomv1beta2.DynamoGraphDeploymentRun{} + if getErr := r.Get(ctx, client.ObjectKeyFromObject(run), existing); getErr != nil { + return ctrl.Result{}, fmt.Errorf("get existing Run %q: %w", wantName, getErr) + } + if metav1.IsControlledBy(existing, request) { + runs = append(runs, *existing) + accepted = true + } else { + condition = dgdrV1beta2Condition(request, metav1.ConditionFalse, dgdrReasonRunNameConflict, + fmt.Sprintf("Run %q exists and is owned by another object; trying again once it is gone", wantName)) + result.RequeueAfter = dgdrV1beta2ConflictRetry + } + default: return ctrl.Result{}, fmt.Errorf("create Run %q: %w", wantName, err) } - runs = append(runs, *run) - accepted = true } } @@ -153,7 +181,7 @@ func (r *DynamoGraphDeploymentRequestV1beta2Reconciler) Reconcile( return ctrl.Result{}, fmt.Errorf("update Request status: %w", err) } } - return ctrl.Result{}, nil + return result, nil } // buildRun resolves the optional inputs of the Request into an immutable Run spec. The effective @@ -162,6 +190,10 @@ func (r *DynamoGraphDeploymentRequestV1beta2Reconciler) Reconcile( func (r *DynamoGraphDeploymentRequestV1beta2Reconciler) buildRun( request *nvidiacomv1beta2.DynamoGraphDeploymentRequest, ) (*nvidiacomv1beta2.DynamoGraphDeploymentRun, error) { + if name := dgdrV1beta2RunName(request); len(name) > dgdrV1beta2MaxRunNameLength { + return nil, fmt.Errorf("the Request name is too long: the Run name %q would exceed %d characters", + name, dgdrV1beta2MaxRunNameLength) + } spec := &request.Spec if spec.AISimulate == nil || spec.AISimulate.Image == "" { return nil, errors.New("spec.aisimulate.image is required") @@ -193,7 +225,7 @@ func (r *DynamoGraphDeploymentRequestV1beta2Reconciler) buildRun( ObjectMeta: metav1.ObjectMeta{ Name: dgdrV1beta2RunName(request), Namespace: request.Namespace, - Labels: map[string]string{dgdrV1beta2RequestLabel: request.Name}, + Labels: map[string]string{dgdrV1beta2RequestLabel: string(request.UID)}, Annotations: map[string]string{dgdrV1beta2GenerationAnnotation: strconv.FormatInt(request.Generation, 10)}, }, Spec: nvidiacomv1beta2.DynamoGraphDeploymentRunSpec{NativeSearchInput: *frozen}, @@ -211,7 +243,7 @@ func (r *DynamoGraphDeploymentRequestV1beta2Reconciler) ownedRuns( ) ([]nvidiacomv1beta2.DynamoGraphDeploymentRun, error) { list := &nvidiacomv1beta2.DynamoGraphDeploymentRunList{} if err := r.List(ctx, list, client.InNamespace(request.Namespace), - client.MatchingLabels{dgdrV1beta2RequestLabel: request.Name}); err != nil { + client.MatchingLabels{dgdrV1beta2RequestLabel: string(request.UID)}); err != nil { return nil, fmt.Errorf("list Runs: %w", err) } owned := make([]nvidiacomv1beta2.DynamoGraphDeploymentRun, 0, len(list.Items)) diff --git a/deploy/operator/internal/controller/dynamographdeploymentrequest_v1beta2_controller_test.go b/deploy/operator/internal/controller/dynamographdeploymentrequest_v1beta2_controller_test.go index 821fdd19f2df..fac81ccf20cd 100644 --- a/deploy/operator/internal/controller/dynamographdeploymentrequest_v1beta2_controller_test.go +++ b/deploy/operator/internal/controller/dynamographdeploymentrequest_v1beta2_controller_test.go @@ -20,6 +20,7 @@ package controller import ( "encoding/json" "fmt" + "strings" "testing" nvidiacomv1beta2 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta2" @@ -277,7 +278,7 @@ func dgdrV1beta2TerminalRun(request *nvidiacomv1beta2.DynamoGraphDeploymentReque ObjectMeta: metav1.ObjectMeta{ Name: fmt.Sprintf("search-%d", generation), Namespace: "default", - Labels: map[string]string{dgdrV1beta2RequestLabel: request.Name}, + Labels: map[string]string{dgdrV1beta2RequestLabel: string(request.UID)}, Annotations: map[string]string{dgdrV1beta2GenerationAnnotation: fmt.Sprint(generation)}, OwnerReferences: []metav1.OwnerReference{{ APIVersion: nvidiacomv1beta2.GroupVersion.String(), @@ -296,3 +297,67 @@ func boolPointerForTest(value bool) *bool { return &value } // apiextensionsJSONForTest is a non-empty v1beta1 carrier payload. var apiextensionsJSONForTest = apiextensionsv1.JSON{Raw: []byte(`{"model":"carrier"}`)} + +func TestV1beta2RequestControllerLabelsRunsWithTheRequestUID(t *testing.T) { + request := dgdrV1beta2Request(true) + request.Name = strings.Repeat("a", 100) + f := newDGDRV1beta2Fixture(t, request) + run, err := f.reconciler.buildRun(request) + if err != nil { + t.Fatalf("build run: %v", err) + } + value := run.Labels[dgdrV1beta2RequestLabel] + if value != string(request.UID) { + t.Fatalf("label = %q, want the request UID", value) + } + if len(value) > 63 { + t.Fatalf("label value has %d characters; a label value is limited to 63", len(value)) + } +} + +func TestV1beta2RequestControllerRejectsANameThatCannotBeDerived(t *testing.T) { + request := dgdrV1beta2Request(true) + request.Name = strings.Repeat("a", 252) + f := newDGDRV1beta2Fixture(t, request) + if _, err := f.reconciler.buildRun(request); err == nil || !strings.Contains(err.Error(), "too long") { + t.Fatalf("err = %v, want a name-too-long error", err) + } +} + +func TestV1beta2RequestControllerDoesNotAdoptARunOfAnotherOwner(t *testing.T) { + stale := &nvidiacomv1beta2.DynamoGraphDeploymentRun{ + ObjectMeta: metav1.ObjectMeta{ + Name: "search-1", + Namespace: "default", + Labels: map[string]string{dgdrV1beta2RequestLabel: "old-uid"}, + OwnerReferences: []metav1.OwnerReference{{ + APIVersion: nvidiacomv1beta2.GroupVersion.String(), + Kind: "DynamoGraphDeploymentRequest", + Name: "search", + UID: "old-uid", + Controller: boolPointerForTest(true), + }}, + }, + } + f := newDGDRV1beta2Fixture(t, dgdrV1beta2Request(true), stale) + + result, err := f.reconciler.Reconcile(t.Context(), ctrl.Request{ + NamespacedName: types.NamespacedName{Namespace: "default", Name: "search"}, + }) + if err != nil { + t.Fatalf("reconcile: %v", err) + } + if result.RequeueAfter <= 0 { + t.Fatal("no retry was scheduled for the Run name held by another owner") + } + request := f.request() + if condition := dgdrV1beta2ReconciledCondition(t, request); condition.Reason != dgdrReasonRunNameConflict { + t.Fatalf("condition = %v, want reason %s", condition, dgdrReasonRunNameConflict) + } + if request.Status.ActiveRunRef != nil { + t.Fatalf("activeRunRef = %v, want none: the old Run is not ours", request.Status.ActiveRunRef) + } + if request.Status.ObservedGeneration != 0 { + t.Fatalf("observedGeneration = %d, want 0", request.Status.ObservedGeneration) + } +} From 01ddc2e626a5b143e923920f7d08c5ebe064c637 Mon Sep 17 00:00:00 2001 From: devivasudevan <49675305+devivasudevan@users.noreply.github.com> Date: Thu, 22 Oct 2026 14:05:39 +0200 Subject: [PATCH 26/40] test(operator): check the run publisher's server-side apply payloads against the v1beta2 CRDs Add an envtest for the publisher's writer: candidate and run-status payloads are accepted by the real schema, the publisher owns the fields it writes, resourceVersion constraints produce conflicts, omitted fields are removed, and candidate deletion honors the UID precondition. Register the package for make envtest and wrap two long lines in the publisher command. Signed-off-by: devivasudevan <49675305+devivasudevan@users.noreply.github.com> --- deploy/operator/Makefile | 2 +- .../operator/cmd/dgdr-run-publisher/main.go | 8 +- .../dgdrrunpublisher/writer_envtest_test.go | 265 ++++++++++++++++++ 3 files changed, 272 insertions(+), 3 deletions(-) create mode 100644 deploy/operator/internal/dgdrrunpublisher/writer_envtest_test.go diff --git a/deploy/operator/Makefile b/deploy/operator/Makefile index 8224c328c29e..bcbcfe7a0942 100644 --- a/deploy/operator/Makefile +++ b/deploy/operator/Makefile @@ -4,7 +4,7 @@ IMG ?= controller:latest # ENVTEST_K8S_VERSION refers to the version of kubebuilder assets to be downloaded by envtest binary. ENVTEST_K8S_VERSION = 1.30.0 -ENVTEST_PACKAGES ?= ./internal/controller/... ./internal/webhook/validation ./cmd/crd-apply +ENVTEST_PACKAGES ?= ./internal/controller/... ./internal/webhook/validation ./cmd/crd-apply ./internal/dgdrrunpublisher ENVTEST_ASSETS_CMD = $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path CLUSTERTEST_PACKAGES ?= ./internal/testing/clusterenv ./internal/controller/... diff --git a/deploy/operator/cmd/dgdr-run-publisher/main.go b/deploy/operator/cmd/dgdr-run-publisher/main.go index 60e55e6cd48b..05c5c8f1cbd8 100644 --- a/deploy/operator/cmd/dgdr-run-publisher/main.go +++ b/deploy/operator/cmd/dgdr-run-publisher/main.go @@ -48,13 +48,17 @@ func run() int { flag.StringVar(&cfg.RunUID, "run-uid", "", "UID of the DynamoGraphDeploymentRun this pod serves") flag.StringVar(&cfg.SweeperContainer, "sweeper-container", "sweeper", "name of the Sweeper container in this pod") flag.DurationVar(&cfg.MinPollInterval, "poll-interval", time.Second, "snapshot check interval after a change") - flag.DurationVar(&cfg.MaxPollInterval, "max-poll-interval", 5*time.Second, "longest snapshot check interval while the snapshot is unchanged") + flag.DurationVar( + &cfg.MaxPollInterval, "max-poll-interval", 5*time.Second, + "longest snapshot check interval while the snapshot is unchanged", + ) flag.Parse() ctrl.SetLogger(zap.New(zap.UseFlagOptions(&logOptions))) cfg.Namespace, cfg.PodName = os.Getenv("POD_NAMESPACE"), os.Getenv("POD_NAME") if cfg.SnapshotDir == "" || cfg.RunName == "" || cfg.RunUID == "" || cfg.Namespace == "" || cfg.PodName == "" { - fmt.Fprintln(os.Stderr, "--snapshot-dir, --run-name, --run-uid and the POD_NAMESPACE/POD_NAME environment variables are required") + fmt.Fprintln(os.Stderr, "--snapshot-dir, --run-name, --run-uid and the POD_NAMESPACE/POD_NAME "+ + "environment variables are required") return dgdrrunpublisher.ExitReconcileFailed } diff --git a/deploy/operator/internal/dgdrrunpublisher/writer_envtest_test.go b/deploy/operator/internal/dgdrrunpublisher/writer_envtest_test.go new file mode 100644 index 000000000000..77552d82ebb7 --- /dev/null +++ b/deploy/operator/internal/dgdrrunpublisher/writer_envtest_test.go @@ -0,0 +1,265 @@ +//go:build !clustertest + +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dgdrrunpublisher + +import ( + "context" + "strings" + "testing" + "time" + + v1beta2 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta2" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/dgdcreconcile" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/testing/operatorenv" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// These tests send the publisher's real server-side-apply payloads to an API server that +// has the v1beta2 CRDs installed. The reconciler tests use a fake writer; this is the +// check that the payloads are accepted by the real schema, that the publisher owns what +// it writes, and that the resourceVersion constraints behave as the reconciler assumes. + +type writerFixture struct { + t *testing.T + ctx context.Context + client client.Client + run *v1beta2.DynamoGraphDeploymentRun + writer *clientWriter +} + +func newWriterFixture(t *testing.T) *writerFixture { + t.Helper() + env := operatorenv.New(operatorenv.Options{ + // The publisher has no webhooks; the environment only requires the hook to exist. + SetupWebhooks: func(ctrl.Manager, operatorenv.WebhookSetupOptions) error { return nil }, + }).RunT(t) + ctx := context.Background() + c := env.Client() + + run := &v1beta2.DynamoGraphDeploymentRun{ + ObjectMeta: metav1.ObjectMeta{Namespace: env.Namespace(), Name: "publisher-run"}, + Spec: v1beta2.DynamoGraphDeploymentRunSpec{ + NativeSearchInput: v1beta2.NativeSearchInput{Model: &v1beta2.ModelSpec{Name: "Qwen/Qwen3-0.6B"}}, + }, + } + if err := c.Create(ctx, run); err != nil { + t.Fatalf("create run: %v", err) + } + return &writerFixture{ + t: t, + ctx: ctx, + client: c, + run: run, + writer: &clientWriter{ + Client: c, + Reader: c, + Namespace: run.Namespace, + RunName: run.Name, + RunUID: string(run.UID), + }, + } +} + +func (f *writerFixture) name(id string) string { + return CandidateName(f.run.Name, string(f.run.UID), id) +} + +func (f *writerFixture) candidate(name string) *v1beta2.DynamoGraphDeploymentCandidate { + f.t.Helper() + var got v1beta2.DynamoGraphDeploymentCandidate + if err := f.client.Get(f.ctx, types.NamespacedName{Namespace: f.run.Namespace, Name: name}, &got); err != nil { + f.t.Fatalf("get candidate %s: %v", name, err) + } + return &got +} + +func (f *writerFixture) currentRun() *v1beta2.DynamoGraphDeploymentRun { + f.t.Helper() + var got v1beta2.DynamoGraphDeploymentRun + if err := f.client.Get(f.ctx, client.ObjectKeyFromObject(f.run), &got); err != nil { + f.t.Fatalf("get run: %v", err) + } + return &got +} + +func envtestCandidate() dgdcreconcile.DesiredCandidate { + return dgdcreconcile.DesiredCandidate{ + ID: "a", + Spec: validManifest + "---\napiVersion: v1\nkind: ConfigMap\nmetadata:\n name: companion\n", + Parameters: map[string]any{"tp": 2}, + Metrics: map[string]any{"throughput": 12.5}, + } +} + +func hasApplyManager(fields []metav1.ManagedFieldsEntry, subresource string) bool { + for _, entry := range fields { + if entry.Manager == fieldOwner && entry.Operation == metav1.ManagedFieldsOperationApply && entry.Subresource == subresource { + return true + } + } + return false +} + +func TestEnvtestCandidateApplyIsAcceptedAndIdempotent(t *testing.T) { + f := newWriterFixture(t) + name := f.name("a") + + stored, err := f.writer.CreateCandidate(f.ctx, f.run, name, envtestCandidate()) + if err != nil { + t.Fatalf("create candidate: %v", err) + } + if stored.ResourceVersion == "" || stored.Populated { + t.Fatalf("stored = %+v, want a resourceVersion and no status yet", stored) + } + + got := f.candidate(name) + if got.Labels[LabelRunUID] != string(f.run.UID) || got.Labels[LabelCandidateID] != "a" { + t.Errorf("labels = %v", got.Labels) + } + if owner := metav1.GetControllerOf(got); owner == nil || owner.UID != f.run.UID || owner.Kind != "DynamoGraphDeploymentRun" { + t.Errorf("controller owner = %+v, want the run", owner) + } + if !strings.Contains(got.Annotations[AnnotationAdditionalResources], "kind: ConfigMap") { + t.Errorf("companion annotation = %q", got.Annotations[AnnotationAdditionalResources]) + } + if !hasApplyManager(got.ManagedFields, "") { + t.Errorf("no apply manager %q on the candidate: %+v", fieldOwner, got.ManagedFields) + } + + again, err := f.writer.CreateCandidate(f.ctx, f.run, name, envtestCandidate()) + if err != nil { + t.Fatalf("re-apply candidate: %v", err) + } + if again.ResourceVersion != stored.ResourceVersion { + t.Errorf("re-applying the same candidate changed resourceVersion %s -> %s", stored.ResourceVersion, again.ResourceVersion) + } +} + +func TestEnvtestCandidateStatusIsWrittenOnceAndGuardedByResourceVersion(t *testing.T) { + f := newWriterFixture(t) + name := f.name("a") + candidate := envtestCandidate() + + stored, err := f.writer.CreateCandidate(f.ctx, f.run, name, candidate) + if err != nil { + t.Fatalf("create candidate: %v", err) + } + if err := f.writer.PopulateStatus(f.ctx, name, stored.ResourceVersion, candidate); err != nil { + t.Fatalf("populate status: %v", err) + } + got := f.candidate(name) + if got.Status.Metrics == nil { + t.Fatal("status.metrics is not set after PopulateStatus") + } + if !hasApplyManager(got.ManagedFields, "status") { + t.Errorf("no status apply manager %q: %+v", fieldOwner, got.ManagedFields) + } + + // A decision made from the version before the status was written must conflict. + err = f.writer.PopulateStatus(f.ctx, name, stored.ResourceVersion, candidate) + if !apierrors.IsConflict(err) { + t.Fatalf("populate with a stale resourceVersion: err = %v, want a conflict", err) + } + + // Re-applying the candidate after its status exists reports it as populated. + after, err := f.writer.CreateCandidate(f.ctx, f.run, name, candidate) + if err != nil { + t.Fatalf("re-apply candidate: %v", err) + } + if !after.Populated { + t.Error("a candidate with status was reported as not populated") + } +} + +func TestEnvtestRunStatusIsOwnedByThePublisher(t *testing.T) { + f := newWriterFixture(t) + first := time.Date(2026, 10, 22, 12, 0, 0, 0, time.UTC) + + staleVersion := f.currentRun().ResourceVersion + if err := f.writer.UpdateRunStatus(f.ctx, staleVersion, RunStatus{ + Published: 2, + LastProgress: first, + CandidateNames: []string{f.name("a"), f.name("b")}, + }); err != nil { + t.Fatalf("update run status: %v", err) + } + run := f.currentRun() + if len(run.Status.CandidateRefs) != 2 || run.Status.CandidateRefs[0].Name != f.name("a") { + t.Fatalf("candidateRefs = %+v", run.Status.CandidateRefs) + } + if run.Status.Progress == nil || run.Status.Progress.Candidates == nil || run.Status.Progress.Candidates.Published != 2 { + t.Fatalf("progress = %+v, want 2 published", run.Status.Progress) + } + if run.Status.LastProgressTime == nil || !run.Status.LastProgressTime.UTC().Equal(first) { + t.Fatalf("lastProgressTime = %v, want %v", run.Status.LastProgressTime, first) + } + if !hasApplyManager(run.ManagedFields, "status") { + t.Errorf("no status apply manager %q: %+v", fieldOwner, run.ManagedFields) + } + + // A write based on the version read before the first write conflicts. + err := f.writer.UpdateRunStatus(f.ctx, staleVersion, RunStatus{Published: 1, CandidateNames: []string{f.name("a")}}) + if !apierrors.IsConflict(err) { + t.Fatalf("update with a stale resourceVersion: err = %v, want a conflict", err) + } + + // Fields the publisher owns and omits are removed: this is why the reconciler carries + // the existing progress time into every payload. + if err := f.writer.UpdateRunStatus(f.ctx, run.ResourceVersion, RunStatus{Published: 1, CandidateNames: []string{f.name("a")}}); err != nil { + t.Fatalf("shrinking update: %v", err) + } + run = f.currentRun() + if len(run.Status.CandidateRefs) != 1 || run.Status.Progress.Candidates.Published != 1 { + t.Errorf("status after shrinking = %+v", run.Status) + } + if run.Status.LastProgressTime != nil { + t.Errorf("lastProgressTime = %v, want it removed when the payload omits it", run.Status.LastProgressTime) + } +} + +func TestEnvtestDeleteCandidateHonorsTheUIDPrecondition(t *testing.T) { + f := newWriterFixture(t) + name := f.name("a") + if _, err := f.writer.CreateCandidate(f.ctx, f.run, name, envtestCandidate()); err != nil { + t.Fatalf("create candidate: %v", err) + } + uid := f.candidate(name).UID + + if err := f.writer.DeleteCandidate(f.ctx, name, types.UID("not-the-uid")); err == nil { + t.Fatal("deleting with the wrong UID succeeded; a recreated candidate would be lost") + } + f.candidate(name) // still there + + if err := f.writer.DeleteCandidate(f.ctx, name, uid); err != nil { + t.Fatalf("delete with the right UID: %v", err) + } + var gone v1beta2.DynamoGraphDeploymentCandidate + err := f.client.Get(f.ctx, types.NamespacedName{Namespace: f.run.Namespace, Name: name}, &gone) + if !apierrors.IsNotFound(err) { + t.Fatalf("candidate still exists after delete: %v", err) + } + if err := f.writer.DeleteCandidate(f.ctx, name, uid); err != nil { + t.Fatalf("deleting a missing candidate must not fail: %v", err) + } +} From 56c96819bc5c0e06f1b8f8e32d108a5d06dc6efc Mon Sep 17 00:00:00 2001 From: devivasudevan <49675305+devivasudevan@users.noreply.github.com> Date: Thu, 22 Oct 2026 16:39:04 +0200 Subject: [PATCH 27/40] fix(operator): address review of the v1beta2 Request controller Publish the Accepted condition, gate the next Run on the Run's Completed condition, use the controller OwnerReference (with a cache index) as the only ownership source, derive the Run generation from the immutable Run name, publish status before deleting retired Runs, and add InputForRun so Run execution reads the frozen input instead of lowering the Run again. Replace the direct-Reconcile envtest with a manager-backed one and cover coalesced generations, null-deleted input keys and status-patch failure. Signed-off-by: devivasudevan <49675305+devivasudevan@users.noreply.github.com> --- .../dynamographdeploymentrun_conditions.go | 23 ++ .../internal/controller/aisimulate/run.go | 55 ++++ .../controller/aisimulate/run_test.go | 58 ++++ ...aphdeploymentrequest_v1beta2_controller.go | 152 ++++++--- ...request_v1beta2_controller_envtest_test.go | 186 ++++++----- ...ploymentrequest_v1beta2_controller_test.go | 306 +++++++++++++++--- 6 files changed, 603 insertions(+), 177 deletions(-) create mode 100644 deploy/operator/api/v1beta2/dynamographdeploymentrun_conditions.go create mode 100644 deploy/operator/internal/controller/aisimulate/run.go create mode 100644 deploy/operator/internal/controller/aisimulate/run_test.go diff --git a/deploy/operator/api/v1beta2/dynamographdeploymentrun_conditions.go b/deploy/operator/api/v1beta2/dynamographdeploymentrun_conditions.go new file mode 100644 index 000000000000..fdc0f0dd1d50 --- /dev/null +++ b/deploy/operator/api/v1beta2/dynamographdeploymentrun_conditions.go @@ -0,0 +1,23 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package v1beta2 + +// DGDRRunConditionCompleted is the condition that makes a Run's outcome authoritative: it is True once +// the Run reached a terminal result. status.phase only describes how a completed Run ended, and +// consumers (such as the Request controller) decide on this condition, not on the phase. +const DGDRRunConditionCompleted = "Completed" diff --git a/deploy/operator/internal/controller/aisimulate/run.go b/deploy/operator/internal/controller/aisimulate/run.go new file mode 100644 index 000000000000..0c6ce9a1f631 --- /dev/null +++ b/deploy/operator/internal/controller/aisimulate/run.go @@ -0,0 +1,55 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package aisimulate + +import ( + "encoding/json" + "errors" + "fmt" + + "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta2" +) + +// InputForRun returns the AISimulate input a Run executes. run.spec.aisimulate.parameters is the +// complete resolved native input, frozen when the Run was created. Run execution must use this +// object directly and must not pass the Run through Request lowering (Lower): the typed fields of +// the Run are copies of the Request and would bring back keys that spec.aisimulate.parameters +// deleted with a JSON merge patch null, and the nulls themselves are no longer in the stored result. +func InputForRun(run *v1beta2.DynamoGraphDeploymentRun) (map[string]any, error) { + if run.Spec.AISimulate == nil || run.Spec.AISimulate.Parameters == nil { + return nil, errors.New("run has no spec.aisimulate.parameters") + } + raw := run.Spec.AISimulate.Parameters.Raw + if len(raw) == 0 && run.Spec.AISimulate.Parameters.Object != nil { + var err error + if raw, err = json.Marshal(run.Spec.AISimulate.Parameters.Object); err != nil { + return nil, fmt.Errorf("run spec.aisimulate.parameters: %w", err) + } + } + if len(raw) == 0 { + return nil, errors.New("run spec.aisimulate.parameters is empty") + } + var input map[string]any + if err := json.Unmarshal(raw, &input); err != nil { + return nil, fmt.Errorf("run spec.aisimulate.parameters must be a JSON object: %w", err) + } + if len(input) == 0 { + return nil, errors.New("run spec.aisimulate.parameters is empty") + } + return input, nil +} diff --git a/deploy/operator/internal/controller/aisimulate/run_test.go b/deploy/operator/internal/controller/aisimulate/run_test.go new file mode 100644 index 000000000000..1373c8c9536c --- /dev/null +++ b/deploy/operator/internal/controller/aisimulate/run_test.go @@ -0,0 +1,58 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package aisimulate + +import ( + "testing" + + "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta2" + "k8s.io/apimachinery/pkg/runtime" +) + +func runWithParameters(raw string) *v1beta2.DynamoGraphDeploymentRun { + run := &v1beta2.DynamoGraphDeploymentRun{} + run.Spec.AISimulate = &v1beta2.AISimulateSpec{Parameters: &runtime.RawExtension{Raw: []byte(raw)}} + return run +} + +func TestInputForRunReturnsTheFrozenObjectUnchanged(t *testing.T) { + input, err := InputForRun(runWithParameters(`{"search_space":{"gpu_budget":4},"workload":{"osl":128}}`)) + if err != nil { + t.Fatalf("InputForRun: %v", err) + } + workload, _ := input["workload"].(map[string]any) + if _, found := workload["isl"]; found || workload["osl"] != float64(128) { + t.Fatalf("workload = %v, want exactly the stored keys", workload) + } +} + +func TestInputForRunRejectsAMissingOrMalformedInput(t *testing.T) { + for name, run := range map[string]*v1beta2.DynamoGraphDeploymentRun{ + "no aisimulate": {}, + "no parameters": {Spec: v1beta2.DynamoGraphDeploymentRunSpec{ + NativeSearchInput: v1beta2.NativeSearchInput{AISimulate: &v1beta2.AISimulateSpec{}}, + }}, + "empty object": runWithParameters(`{}`), + "not an object": runWithParameters(`[1]`), + "not json": runWithParameters(`{`), + } { + if _, err := InputForRun(run); err == nil { + t.Errorf("%s: want an error", name) + } + } +} diff --git a/deploy/operator/internal/controller/dynamographdeploymentrequest_v1beta2_controller.go b/deploy/operator/internal/controller/dynamographdeploymentrequest_v1beta2_controller.go index 3cab7bb4075f..5fa7f6e75133 100644 --- a/deploy/operator/internal/controller/dynamographdeploymentrequest_v1beta2_controller.go +++ b/deploy/operator/internal/controller/dynamographdeploymentrequest_v1beta2_controller.go @@ -24,6 +24,7 @@ import ( "fmt" "sort" "strconv" + "strings" "time" configv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/config/v1alpha1" @@ -45,17 +46,20 @@ import ( ) const ( - // dgdrV1beta2RequestLabel holds the UID of the Request that owns a Run. A label value is limited to - // 63 characters, which a Request name is not, so the UID is used. - dgdrV1beta2RequestLabel = "nvidia.com/dgdr-request-uid" - // dgdrV1beta2GenerationAnnotation records the Request generation a Run was created for. + // dgdrV1beta2GenerationAnnotation records the Request generation a Run was created for. It is + // diagnostic only: the generation the controller acts on is the one in the immutable Run name. dgdrV1beta2GenerationAnnotation = "nvidia.com/dgdr-request-generation" + // dgdrRunControllerUIDIndex indexes Runs by the UID of their controller OwnerReference. The + // OwnerReference is the only definition of which Runs belong to a Request. + dgdrRunControllerUIDIndex = "dynamo.nvidia.com/dgdr-run-controller-uid" + // dgdrV1beta2RetainedRuns is how many terminal Runs a Request keeps. Older ones are deleted. dgdrV1beta2RetainedRuns = 5 - // DGDRReconciledConditionV1beta2 reports whether the controller accepted the current generation. - DGDRReconciledConditionV1beta2 = "Reconciled" + // DGDRAcceptedConditionV1beta2 reports whether the current Request generation has been + // materialized as an immutable Run. Execution state belongs to the Run, not to this condition. + DGDRAcceptedConditionV1beta2 = "Accepted" dgdrReasonRunCreated = "RunCreated" dgdrReasonInvalidSpec = "InvalidSpec" @@ -67,9 +71,6 @@ const ( // dgdrV1beta2ConflictRetry is how long to wait before looking again at a Run name held by another owner. dgdrV1beta2ConflictRetry = 5 * time.Second - - dgdrRunPhaseSucceeded = "Succeeded" - dgdrRunPhaseFailed = "Failed" ) // DynamoGraphDeploymentRequestV1beta2Reconciler turns an accepted native v1beta2 Request into an @@ -85,8 +86,9 @@ type DynamoGraphDeploymentRequestV1beta2Reconciler struct { // +kubebuilder:rbac:groups=nvidia.com,resources=dynamographdeploymentrequests/status,verbs=get;update;patch // +kubebuilder:rbac:groups=nvidia.com,resources=dynamographdeploymentruns,verbs=get;list;watch;create;delete -// Reconcile creates the Run for the current generation of a native v1beta2 Request once no other Run -// is active, and records the active and retained Runs in the Request status. +// Reconcile is level based: once no older Run is active it creates the Run of the latest generation +// only, so generations edited in between are coalesced and never get a Run of their own. It records +// the active and retained Runs in the Request status before it deletes any retired Run. func (r *DynamoGraphDeploymentRequestV1beta2Reconciler) Reconcile( ctx context.Context, req ctrl.Request, @@ -117,11 +119,16 @@ func (r *DynamoGraphDeploymentRequestV1beta2Reconciler) Reconcile( } } + active, _, err := dgdrV1beta2SplitRuns(request.Name, runs) + if err != nil { + return ctrl.Result{}, err + } + accepted := current != nil condition := dgdrV1beta2Condition(request, metav1.ConditionTrue, dgdrReasonRunCreated, fmt.Sprintf("Run %q exists for generation %d", wantName, request.Generation)) if current == nil { - if active := dgdrV1beta2NewestActive(runs); active != nil { + if active != nil { condition = dgdrV1beta2Condition(request, metav1.ConditionFalse, dgdrReasonWaitingForActiveRun, fmt.Sprintf("Run %q is still active; the Run for generation %d starts when it finishes", active.Name, request.Generation)) @@ -155,13 +162,12 @@ func (r *DynamoGraphDeploymentRequestV1beta2Reconciler) Reconcile( } } - active, terminal := dgdrV1beta2SplitRuns(runs) - keep := min(len(terminal), dgdrV1beta2RetainedRuns) - for i := keep; i < len(terminal); i++ { - if err := r.Delete(ctx, &terminal[i]); err != nil && !apierrors.IsNotFound(err) { - return ctrl.Result{}, fmt.Errorf("delete retired Run %q: %w", terminal[i].Name, err) - } + active, terminal, err := dgdrV1beta2SplitRuns(request.Name, runs) + if err != nil { + return ctrl.Result{}, err } + keep := min(len(terminal), dgdrV1beta2RetainedRuns) + retired := terminal[keep:] if accepted { request.Status.ObservedGeneration = request.Generation @@ -181,12 +187,21 @@ func (r *DynamoGraphDeploymentRequestV1beta2Reconciler) Reconcile( return ctrl.Result{}, fmt.Errorf("update Request status: %w", err) } } + + // The status no longer references these Runs, so they may go. If a deletion fails, an unreferenced + // child stays for the next pass; the status never points at a Run that was already deleted. + for i := range retired { + if err := r.Delete(ctx, &retired[i]); err != nil && !apierrors.IsNotFound(err) { + return ctrl.Result{}, fmt.Errorf("delete retired Run %q: %w", retired[i].Name, err) + } + } return result, nil } // buildRun resolves the optional inputs of the Request into an immutable Run spec. The effective -// AISimulate input is frozen into aisimulate.parameters, which is authoritative, so lowering the Run -// again yields the same bytes. +// AISimulate input is frozen into aisimulate.parameters: that object is the complete resolved input. +// Run execution reads it with aisimulate.InputForRun and never lowers the Run again, because the +// merge-patch deletions (null) that shaped it are not part of the stored result. func (r *DynamoGraphDeploymentRequestV1beta2Reconciler) buildRun( request *nvidiacomv1beta2.DynamoGraphDeploymentRequest, ) (*nvidiacomv1beta2.DynamoGraphDeploymentRun, error) { @@ -225,7 +240,6 @@ func (r *DynamoGraphDeploymentRequestV1beta2Reconciler) buildRun( ObjectMeta: metav1.ObjectMeta{ Name: dgdrV1beta2RunName(request), Namespace: request.Namespace, - Labels: map[string]string{dgdrV1beta2RequestLabel: string(request.UID)}, Annotations: map[string]string{dgdrV1beta2GenerationAnnotation: strconv.FormatInt(request.Generation, 10)}, }, Spec: nvidiacomv1beta2.DynamoGraphDeploymentRunSpec{NativeSearchInput: *frozen}, @@ -236,14 +250,15 @@ func (r *DynamoGraphDeploymentRequestV1beta2Reconciler) buildRun( return run, nil } -// ownedRuns returns the Runs this Request controls. +// ownedRuns returns the Runs this Request controls. The controller OwnerReference is the only +// source of ownership: the cache index selects candidates and IsControlledBy is the exact check. func (r *DynamoGraphDeploymentRequestV1beta2Reconciler) ownedRuns( ctx context.Context, request *nvidiacomv1beta2.DynamoGraphDeploymentRequest, ) ([]nvidiacomv1beta2.DynamoGraphDeploymentRun, error) { list := &nvidiacomv1beta2.DynamoGraphDeploymentRunList{} if err := r.List(ctx, list, client.InNamespace(request.Namespace), - client.MatchingLabels{dgdrV1beta2RequestLabel: string(request.UID)}); err != nil { + client.MatchingFields{dgdrRunControllerUIDIndex: string(request.UID)}); err != nil { return nil, fmt.Errorf("list Runs: %w", err) } owned := make([]nvidiacomv1beta2.DynamoGraphDeploymentRun, 0, len(list.Items)) @@ -258,6 +273,14 @@ func (r *DynamoGraphDeploymentRequestV1beta2Reconciler) ownedRuns( // SetupWithManager registers the controller. It watches the Runs it owns, so a finished Run wakes // the Request that was waiting for it. func (r *DynamoGraphDeploymentRequestV1beta2Reconciler) SetupWithManager(mgr ctrl.Manager) error { + if err := mgr.GetFieldIndexer().IndexField( + context.Background(), + &nvidiacomv1beta2.DynamoGraphDeploymentRun{}, + dgdrRunControllerUIDIndex, + dgdrRunControllerUIDs, + ); err != nil { + return fmt.Errorf("index Runs by controller UID: %w", err) + } return ctrl.NewControllerManagedBy(mgr). Named("dynamographdeploymentrequest-v1beta2"). For(&nvidiacomv1beta2.DynamoGraphDeploymentRequest{}). @@ -266,6 +289,15 @@ func (r *DynamoGraphDeploymentRequestV1beta2Reconciler) SetupWithManager(mgr ctr Complete(r) } +// dgdrRunControllerUIDs is the index function behind dgdrRunControllerUIDIndex. +func dgdrRunControllerUIDs(obj client.Object) []string { + owner := metav1.GetControllerOf(obj) + if owner == nil { + return nil + } + return []string{string(owner.UID)} +} + // dgdrV1beta2RunName names the Run of one Request generation. A metadata-only update keeps the // generation, so it never creates a second Run. func dgdrV1beta2RunName(request *nvidiacomv1beta2.DynamoGraphDeploymentRequest) string { @@ -278,7 +310,7 @@ func dgdrV1beta2Condition( reason, message string, ) metav1.Condition { return metav1.Condition{ - Type: DGDRReconciledConditionV1beta2, + Type: DGDRAcceptedConditionV1beta2, Status: status, Reason: reason, Message: message, @@ -286,54 +318,70 @@ func dgdrV1beta2Condition( } } +// dgdrV1beta2IsTerminal reports whether the Run has published its Completed condition. The conditions +// are authoritative; the phase only describes how a completed Run ended. func dgdrV1beta2IsTerminal(run *nvidiacomv1beta2.DynamoGraphDeploymentRun) bool { - phase := string(run.Status.Phase) - return phase == dgdrRunPhaseSucceeded || phase == dgdrRunPhaseFailed + return meta.IsStatusConditionTrue(run.Status.Conditions, nvidiacomv1beta2.DGDRRunConditionCompleted) } -// dgdrV1beta2Newer orders Runs newest first: by Request generation, then creation time, then name. -func dgdrV1beta2Newer(a, b *nvidiacomv1beta2.DynamoGraphDeploymentRun) bool { - ga, gb := dgdrV1beta2RunGeneration(a), dgdrV1beta2RunGeneration(b) - if ga != gb { - return ga > gb +// dgdrV1beta2RunGeneration reads the Request generation from the immutable Run name +// (-). A Run that does not follow it is an error, never generation zero. +func dgdrV1beta2RunGeneration(requestName string, run *nvidiacomv1beta2.DynamoGraphDeploymentRun) (int64, error) { + prefix := requestName + "-" + value, ok := strings.CutPrefix(run.Name, prefix) + if !ok { + return 0, fmt.Errorf("run %q does not use the Request prefix %q", run.Name, prefix) } - if !a.CreationTimestamp.Equal(&b.CreationTimestamp) { - return b.CreationTimestamp.Before(&a.CreationTimestamp) + generation, err := strconv.ParseInt(value, 10, 64) + if err != nil { + return 0, fmt.Errorf("run %q has no valid Request generation: %w", run.Name, err) } - return a.Name > b.Name + return generation, nil } -func dgdrV1beta2RunGeneration(run *nvidiacomv1beta2.DynamoGraphDeploymentRun) int64 { - generation, _ := strconv.ParseInt(run.Annotations[dgdrV1beta2GenerationAnnotation], 10, 64) - return generation +type dgdrV1beta2RunEntry struct { + run nvidiacomv1beta2.DynamoGraphDeploymentRun + generation int64 +} + +// dgdrV1beta2Newer orders Runs newest first: by Request generation, then creation time, then name. +func dgdrV1beta2Newer(a, b *dgdrV1beta2RunEntry) bool { + if a.generation != b.generation { + return a.generation > b.generation + } + if !a.run.CreationTimestamp.Equal(&b.run.CreationTimestamp) { + return b.run.CreationTimestamp.Before(&a.run.CreationTimestamp) + } + return a.run.Name > b.run.Name } // dgdrV1beta2SplitRuns returns the newest non-terminal Run, if any, and the terminal Runs newest first. func dgdrV1beta2SplitRuns( + requestName string, runs []nvidiacomv1beta2.DynamoGraphDeploymentRun, -) (*nvidiacomv1beta2.DynamoGraphDeploymentRun, []nvidiacomv1beta2.DynamoGraphDeploymentRun) { - sorted := append([]nvidiacomv1beta2.DynamoGraphDeploymentRun(nil), runs...) - sort.SliceStable(sorted, func(i, j int) bool { return dgdrV1beta2Newer(&sorted[i], &sorted[j]) }) +) (*nvidiacomv1beta2.DynamoGraphDeploymentRun, []nvidiacomv1beta2.DynamoGraphDeploymentRun, error) { + entries := make([]dgdrV1beta2RunEntry, 0, len(runs)) + for i := range runs { + generation, err := dgdrV1beta2RunGeneration(requestName, &runs[i]) + if err != nil { + return nil, nil, err + } + entries = append(entries, dgdrV1beta2RunEntry{run: runs[i], generation: generation}) + } + sort.SliceStable(entries, func(i, j int) bool { return dgdrV1beta2Newer(&entries[i], &entries[j]) }) var active *nvidiacomv1beta2.DynamoGraphDeploymentRun var terminal []nvidiacomv1beta2.DynamoGraphDeploymentRun - for i := range sorted { + for i := range entries { switch { - case !dgdrV1beta2IsTerminal(&sorted[i]): + case !dgdrV1beta2IsTerminal(&entries[i].run): if active == nil { - active = &sorted[i] + active = &entries[i].run } default: - terminal = append(terminal, sorted[i]) + terminal = append(terminal, entries[i].run) } } - return active, terminal -} - -func dgdrV1beta2NewestActive( - runs []nvidiacomv1beta2.DynamoGraphDeploymentRun, -) *nvidiacomv1beta2.DynamoGraphDeploymentRun { - active, _ := dgdrV1beta2SplitRuns(runs) - return active + return active, terminal, nil } // dgdrV1beta2HasEffectiveValue reports whether input holds a non-null, non-empty value at path. diff --git a/deploy/operator/internal/controller/dynamographdeploymentrequest_v1beta2_controller_envtest_test.go b/deploy/operator/internal/controller/dynamographdeploymentrequest_v1beta2_controller_envtest_test.go index 8db3a12f80b1..113c6160a218 100644 --- a/deploy/operator/internal/controller/dynamographdeploymentrequest_v1beta2_controller_envtest_test.go +++ b/deploy/operator/internal/controller/dynamographdeploymentrequest_v1beta2_controller_envtest_test.go @@ -21,10 +21,11 @@ package controller import ( "context" - "strconv" + "testing" + "time" nvidiacomv1beta2 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta2" - . "github.com/onsi/ginkgo/v2" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/controller/aisimulate" . "github.com/onsi/gomega" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -33,90 +34,121 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" ) -var _ = Describe("DynamoGraphDeploymentRequest v1beta2 controller", func() { - var reconciler *DynamoGraphDeploymentRequestV1beta2Reconciler +// This test runs the controller under a real manager, so only the watch events (a Request edit, a Run +// status update, a deleted Run) can make it reconcile. It fails if SetupWithManager loses the Owns +// watch, an event predicate drops Run status updates, or the owner mapping breaks. It also covers the +// API server and admission side: the Request is created model-only and admission supplies the image +// and the hardware. +func TestDGDRV1beta2ControllerReactsToRequestAndRunEvents(t *testing.T) { + g := NewWithT(t) + env := sharedEnv.ForTest(t) + c := env.Client() + ctx := context.Background() - BeforeEach(func() { - reconciler = &DynamoGraphDeploymentRequestV1beta2Reconciler{Client: k8sClient} + env.StartManager(func(mgr ctrl.Manager) error { + return (&DynamoGraphDeploymentRequestV1beta2Reconciler{ + Client: mgr.GetClient(), + Config: env.OperatorConfig(), + RuntimeConfig: env.RuntimeConfig(), + }).SetupWithManager(mgr) }) - reconcile := func(ctx context.Context, key types.NamespacedName) { - _, err := reconciler.Reconcile(ctx, ctrl.Request{NamespacedName: key}) - Expect(err).NotTo(HaveOccurred()) - } + const timeout = 30 * time.Second + const poll = 200 * time.Millisecond - It("Should create one Run per generation through the API server", func() { - ctx := context.Background() - dgdr := &nvidiacomv1beta2.DynamoGraphDeploymentRequest{ - ObjectMeta: metav1.ObjectMeta{Name: "v1beta2-run-per-generation", Namespace: envtestNamespace}, - Spec: nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec{ - NativeSearchInput: nvidiacomv1beta2.NativeSearchInput{ - Model: &nvidiacomv1beta2.ModelSpec{Name: "Qwen/Qwen3-32B"}, - }, + request := &nvidiacomv1beta2.DynamoGraphDeploymentRequest{ + ObjectMeta: metav1.ObjectMeta{Name: "watched", Namespace: env.Namespace()}, + Spec: nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec{ + NativeSearchInput: nvidiacomv1beta2.NativeSearchInput{ + Model: &nvidiacomv1beta2.ModelSpec{Name: "Qwen/Qwen3-32B"}, }, - } - - By("Creating a native v1beta2 request; admission supplies the image and the hardware") - Expect(k8sClient.Create(ctx, dgdr)).Should(Succeed()) - defer func() { _ = k8sClient.Delete(ctx, dgdr) }() - key := client.ObjectKeyFromObject(dgdr) - - By("Reconciling creates the Run of generation 1") - reconcile(ctx, key) + }, + } + key := client.ObjectKeyFromObject(request) + getRequest := func(g Gomega) *nvidiacomv1beta2.DynamoGraphDeploymentRequest { live := &nvidiacomv1beta2.DynamoGraphDeploymentRequest{} - Expect(k8sClient.Get(ctx, key, live)).Should(Succeed()) - firstName := dgdr.Name + "-" + strconv.FormatInt(live.Generation, 10) - first := &nvidiacomv1beta2.DynamoGraphDeploymentRun{} - Expect(k8sClient.Get(ctx, types.NamespacedName{Namespace: envtestNamespace, Name: firstName}, first)).Should(Succeed()) - Expect(metav1.IsControlledBy(first, live)).To(BeTrue()) - Expect(first.Spec.Hardware).NotTo(BeNil()) - Expect(first.Spec.AISimulate.Image).To(Equal(controllerAISimulateImage)) - Expect(first.Spec.AISimulate.VersionOverride).To(Equal("1.2.0")) - Expect(first.Spec.AISimulate.Parameters).NotTo(BeNil()) + g.Expect(c.Get(ctx, key, live)).To(Succeed()) + return live + } + accepted := func(live *nvidiacomv1beta2.DynamoGraphDeploymentRequest) *metav1.Condition { + return meta.FindStatusCondition(live.Status.Conditions, DGDRAcceptedConditionV1beta2) + } + getRun := func(g Gomega, name string) *nvidiacomv1beta2.DynamoGraphDeploymentRun { + run := &nvidiacomv1beta2.DynamoGraphDeploymentRun{} + g.Expect(c.Get(ctx, types.NamespacedName{Namespace: env.Namespace(), Name: name}, run)).To(Succeed()) + return run + } - By("Reading the status the controller wrote through the status subresource") - Expect(live.Status.ObservedGeneration).To(Equal(live.Generation)) - Expect(live.Status.ActiveRunRef).NotTo(BeNil()) - Expect(live.Status.ActiveRunRef.Name).To(Equal(firstName)) - condition := meta.FindStatusCondition(live.Status.Conditions, DGDRReconciledConditionV1beta2) - Expect(condition).NotTo(BeNil()) - Expect(condition.Status).To(Equal(metav1.ConditionTrue)) - Expect(live.Status.V1Beta1).To(BeNil()) + t.Log("Creating the Request produces its first Run without a direct Reconcile call") + g.Expect(c.Create(ctx, request)).To(Succeed()) + var firstName string + g.Eventually(func(g Gomega) { + live := getRequest(g) + g.Expect(live.Status.ActiveRunRef).NotTo(BeNil()) + g.Expect(accepted(live)).NotTo(BeNil()) + g.Expect(accepted(live).Status).To(Equal(metav1.ConditionTrue)) + g.Expect(live.Status.ObservedGeneration).To(Equal(live.Generation)) + firstName = live.Status.ActiveRunRef.Name + }).WithTimeout(timeout).WithPolling(poll).Should(Succeed()) - By("Reconciling again creates nothing") - reconcile(ctx, key) - runs := &nvidiacomv1beta2.DynamoGraphDeploymentRunList{} - Expect(k8sClient.List(ctx, runs, client.InNamespace(envtestNamespace))).Should(Succeed()) - Expect(runs.Items).To(HaveLen(1)) + first := getRun(g, firstName) + g.Expect(metav1.IsControlledBy(first, getRequest(g))).To(BeTrue()) + g.Expect(first.Spec.Hardware).NotTo(BeNil()) + g.Expect(first.Spec.AISimulate.Image).To(Equal(controllerAISimulateImage)) + g.Expect(first.Spec.AISimulate.VersionOverride).To(Equal("1.2.0")) + input, err := aisimulate.InputForRun(first) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(input).To(HaveKey("search_space")) - By("Changing the spec while the Run is active starts nothing and says why") - Expect(k8sClient.Get(ctx, key, live)).Should(Succeed()) + t.Log("Editing the Request while its Run is active is reported and starts nothing") + g.Eventually(func(g Gomega) { + live := getRequest(g) live.Spec.Model.Name = "Qwen/Qwen3-30B-A3B" - Expect(k8sClient.Update(ctx, live)).Should(Succeed()) - reconcile(ctx, key) - Expect(k8sClient.List(ctx, runs, client.InNamespace(envtestNamespace))).Should(Succeed()) - Expect(runs.Items).To(HaveLen(1)) - Expect(k8sClient.Get(ctx, key, live)).Should(Succeed()) - Expect(live.Generation).To(BeNumerically(">", 1)) - condition = meta.FindStatusCondition(live.Status.Conditions, DGDRReconciledConditionV1beta2) - Expect(condition).NotTo(BeNil()) - Expect(condition.Reason).To(Equal(dgdrReasonWaitingForActiveRun)) + g.Expect(c.Update(ctx, live)).To(Succeed()) + }).WithTimeout(timeout).WithPolling(poll).Should(Succeed()) + g.Eventually(func(g Gomega) { + live := getRequest(g) + condition := accepted(live) + g.Expect(condition).NotTo(BeNil()) + g.Expect(condition.Status).To(Equal(metav1.ConditionFalse)) + g.Expect(condition.Reason).To(Equal(dgdrReasonWaitingForActiveRun)) + g.Expect(condition.ObservedGeneration).To(Equal(live.Generation)) + }).WithTimeout(timeout).WithPolling(poll).Should(Succeed()) - By("Finishing the first Run lets the next generation start") - Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(first), first)).Should(Succeed()) - first.Status.Phase = nvidiacomv1beta2.DGDRRunPhase(dgdrRunPhaseSucceeded) - Expect(k8sClient.Status().Update(ctx, first)).Should(Succeed()) - reconcile(ctx, key) + t.Log("Only the Run status update wakes the Request: Completed releases the next generation") + g.Eventually(func(g Gomega) { + run := getRun(g, firstName) + run.Status.Phase = nvidiacomv1beta2.DGDRRunPhase("Succeeded") + meta.SetStatusCondition(&run.Status.Conditions, metav1.Condition{ + Type: nvidiacomv1beta2.DGDRRunConditionCompleted, + Status: metav1.ConditionTrue, + Reason: "Finished", + }) + g.Expect(c.Status().Update(ctx, run)).To(Succeed()) + }).WithTimeout(timeout).WithPolling(poll).Should(Succeed()) - Expect(k8sClient.Get(ctx, key, live)).Should(Succeed()) - secondName := dgdr.Name + "-" + strconv.FormatInt(live.Generation, 10) - Expect(live.Status.ObservedGeneration).To(Equal(live.Generation)) - Expect(live.Status.ActiveRunRef).NotTo(BeNil()) - Expect(live.Status.ActiveRunRef.Name).To(Equal(secondName)) - Expect(live.Status.RecentRunRefs).To(HaveLen(1)) - Expect(live.Status.RecentRunRefs[0].Name).To(Equal(firstName)) - second := &nvidiacomv1beta2.DynamoGraphDeploymentRun{} - Expect(k8sClient.Get(ctx, types.NamespacedName{Namespace: envtestNamespace, Name: secondName}, second)).Should(Succeed()) - Expect(second.Spec.Model.Name).To(Equal("Qwen/Qwen3-30B-A3B")) - }) -}) + var secondName string + var secondUID types.UID + g.Eventually(func(g Gomega) { + live := getRequest(g) + g.Expect(live.Status.ActiveRunRef).NotTo(BeNil()) + g.Expect(live.Status.ActiveRunRef.Name).NotTo(Equal(firstName)) + g.Expect(live.Status.ObservedGeneration).To(Equal(live.Generation)) + g.Expect(accepted(live).Status).To(Equal(metav1.ConditionTrue)) + g.Expect(live.Status.RecentRunRefs).To(HaveLen(1)) + g.Expect(live.Status.RecentRunRefs[0].Name).To(Equal(firstName)) + second := getRun(g, live.Status.ActiveRunRef.Name) + g.Expect(second.Spec.Model.Name).To(Equal("Qwen/Qwen3-30B-A3B")) + secondName, secondUID = second.Name, second.UID + }).WithTimeout(timeout).WithPolling(poll).Should(Succeed()) + + t.Log("Deleting the active Run wakes the Request through the owner mapping and it is recreated") + g.Expect(c.Delete(ctx, getRun(g, secondName))).To(Succeed()) + g.Eventually(func(g Gomega) { + recreated := getRun(g, secondName) + g.Expect(recreated.UID).NotTo(Equal(secondUID)) + live := getRequest(g) + g.Expect(live.Status.ActiveRunRef).NotTo(BeNil()) + g.Expect(live.Status.ActiveRunRef.Name).To(Equal(secondName)) + }).WithTimeout(timeout).WithPolling(poll).Should(Succeed()) +} diff --git a/deploy/operator/internal/controller/dynamographdeploymentrequest_v1beta2_controller_test.go b/deploy/operator/internal/controller/dynamographdeploymentrequest_v1beta2_controller_test.go index fac81ccf20cd..5c6e1ee0a9d6 100644 --- a/deploy/operator/internal/controller/dynamographdeploymentrequest_v1beta2_controller_test.go +++ b/deploy/operator/internal/controller/dynamographdeploymentrequest_v1beta2_controller_test.go @@ -18,12 +18,15 @@ package controller import ( + "context" "encoding/json" + "errors" "fmt" "strings" "testing" nvidiacomv1beta2 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta2" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/controller/aisimulate" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" @@ -33,6 +36,7 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" ) type dgdrV1beta2Fixture struct { @@ -42,6 +46,15 @@ type dgdrV1beta2Fixture struct { } func newDGDRV1beta2Fixture(t *testing.T, objects ...client.Object) *dgdrV1beta2Fixture { + t.Helper() + return newDGDRV1beta2FixtureWithInterceptor(t, interceptor.Funcs{}, objects...) +} + +func newDGDRV1beta2FixtureWithInterceptor( + t *testing.T, + funcs interceptor.Funcs, + objects ...client.Object, +) *dgdrV1beta2Fixture { t.Helper() scheme := runtime.NewScheme() if err := nvidiacomv1beta2.AddToScheme(scheme); err != nil { @@ -50,20 +63,28 @@ func newDGDRV1beta2Fixture(t *testing.T, objects ...client.Object) *dgdrV1beta2F c := fake.NewClientBuilder(). WithScheme(scheme). WithObjects(objects...). + WithIndex(&nvidiacomv1beta2.DynamoGraphDeploymentRun{}, dgdrRunControllerUIDIndex, dgdrRunControllerUIDs). WithStatusSubresource(&nvidiacomv1beta2.DynamoGraphDeploymentRequest{}, &nvidiacomv1beta2.DynamoGraphDeploymentRun{}). + WithInterceptorFuncs(funcs). Build() return &dgdrV1beta2Fixture{t: t, client: c, reconciler: &DynamoGraphDeploymentRequestV1beta2Reconciler{Client: c}} } func (f *dgdrV1beta2Fixture) reconcile() { f.t.Helper() - if _, err := f.reconciler.Reconcile(f.t.Context(), ctrl.Request{ - NamespacedName: types.NamespacedName{Namespace: "default", Name: "search"}, - }); err != nil { + if err := f.tryReconcile(); err != nil { f.t.Fatalf("reconcile: %v", err) } } +func (f *dgdrV1beta2Fixture) tryReconcile() error { + f.t.Helper() + _, err := f.reconciler.Reconcile(f.t.Context(), ctrl.Request{ + NamespacedName: types.NamespacedName{Namespace: "default", Name: "search"}, + }) + return err +} + func (f *dgdrV1beta2Fixture) request() *nvidiacomv1beta2.DynamoGraphDeploymentRequest { f.t.Helper() request := &nvidiacomv1beta2.DynamoGraphDeploymentRequest{} @@ -86,18 +107,49 @@ func (f *dgdrV1beta2Fixture) runNames() []string { return names } -func (f *dgdrV1beta2Fixture) setPhase(name, phase string) { +func (f *dgdrV1beta2Fixture) updateRunStatus(name string, mutate func(*nvidiacomv1beta2.DynamoGraphDeploymentRunStatus)) { f.t.Helper() run := &nvidiacomv1beta2.DynamoGraphDeploymentRun{} if err := f.client.Get(f.t.Context(), types.NamespacedName{Namespace: "default", Name: name}, run); err != nil { f.t.Fatalf("get run %s: %v", name, err) } - run.Status.Phase = nvidiacomv1beta2.DGDRRunPhase(phase) + mutate(&run.Status) if err := f.client.Status().Update(f.t.Context(), run); err != nil { - f.t.Fatalf("set phase of run %s: %v", name, err) + f.t.Fatalf("update status of run %s: %v", name, err) } } +// finish ends a Run the way the Run controller does: it publishes Completed and the outcome phase. +func (f *dgdrV1beta2Fixture) finish(name, phase string) { + f.t.Helper() + f.updateRunStatus(name, func(status *nvidiacomv1beta2.DynamoGraphDeploymentRunStatus) { + status.Phase = nvidiacomv1beta2.DGDRRunPhase(phase) + setCompletedForTest(status) + }) +} + +// setPhaseOnly writes a terminal phase without the Completed condition. +func (f *dgdrV1beta2Fixture) setPhaseOnly(name, phase string) { + f.t.Helper() + f.updateRunStatus(name, func(status *nvidiacomv1beta2.DynamoGraphDeploymentRunStatus) { + status.Phase = nvidiacomv1beta2.DGDRRunPhase(phase) + }) +} + +// setCompletedOnly publishes Completed without a terminal phase. +func (f *dgdrV1beta2Fixture) setCompletedOnly(name string) { + f.t.Helper() + f.updateRunStatus(name, setCompletedForTest) +} + +func setCompletedForTest(status *nvidiacomv1beta2.DynamoGraphDeploymentRunStatus) { + meta.SetStatusCondition(&status.Conditions, metav1.Condition{ + Type: nvidiacomv1beta2.DGDRRunConditionCompleted, + Status: metav1.ConditionTrue, + Reason: "Finished", + }) +} + func (f *dgdrV1beta2Fixture) bumpGeneration(generation int64) { f.t.Helper() request := f.request() @@ -125,11 +177,11 @@ func dgdrV1beta2Request(withHardware bool) *nvidiacomv1beta2.DynamoGraphDeployme return request } -func dgdrV1beta2ReconciledCondition(t *testing.T, request *nvidiacomv1beta2.DynamoGraphDeploymentRequest) metav1.Condition { +func dgdrV1beta2AcceptedCondition(t *testing.T, request *nvidiacomv1beta2.DynamoGraphDeploymentRequest) metav1.Condition { t.Helper() - condition := meta.FindStatusCondition(request.Status.Conditions, DGDRReconciledConditionV1beta2) + condition := meta.FindStatusCondition(request.Status.Conditions, DGDRAcceptedConditionV1beta2) if condition == nil { - t.Fatal("no Reconciled condition") + t.Fatal("no Accepted condition") } return *condition } @@ -149,9 +201,9 @@ func TestV1beta2RequestControllerCreatesTheRunOfTheCurrentGeneration(t *testing. if got := run.Spec.AISimulate.VersionOverride; got != "0.13.0" { t.Fatalf("run versionOverride = %q, want the resolved 0.13.0", got) } - var parameters map[string]any - if err := json.Unmarshal(run.Spec.AISimulate.Parameters.Raw, ¶meters); err != nil { - t.Fatalf("decode frozen parameters: %v", err) + parameters, err := aisimulate.InputForRun(run) + if err != nil { + t.Fatalf("input of the run: %v", err) } searchSpace, _ := parameters["search_space"].(map[string]any) if searchSpace["hardware_sku"] != "h100_sxm" || searchSpace["gpu_budget"] != float64(4) { @@ -166,8 +218,8 @@ func TestV1beta2RequestControllerCreatesTheRunOfTheCurrentGeneration(t *testing. if request.Status.ActiveRunRef == nil || request.Status.ActiveRunRef.Name != "search-1" { t.Fatalf("activeRunRef = %v, want search-1", request.Status.ActiveRunRef) } - if condition := dgdrV1beta2ReconciledCondition(t, request); condition.Status != metav1.ConditionTrue { - t.Fatalf("Reconciled = %v, want True", condition) + if condition := dgdrV1beta2AcceptedCondition(t, request); condition.Status != metav1.ConditionTrue { + t.Fatalf("Accepted = %v, want True", condition) } } @@ -195,12 +247,13 @@ func TestV1beta2RequestControllerWaitsForTheActiveRunBeforeStartingTheNext(t *te if request.Status.ObservedGeneration != 1 { t.Fatalf("observedGeneration = %d, want 1 until generation 2 is accepted", request.Status.ObservedGeneration) } - if condition := dgdrV1beta2ReconciledCondition(t, request); condition.Reason != dgdrReasonWaitingForActiveRun { - t.Fatalf("condition = %v, want reason %s", condition, dgdrReasonWaitingForActiveRun) + condition := dgdrV1beta2AcceptedCondition(t, request) + if condition.Status != metav1.ConditionFalse || condition.Reason != dgdrReasonWaitingForActiveRun { + t.Fatalf("condition = %v, want False/%s", condition, dgdrReasonWaitingForActiveRun) } - t.Log("Once search-1 is terminal, search-2 is created and search-1 moves to the history") - f.setPhase("search-1", dgdrRunPhaseSucceeded) + t.Log("Once search-1 is completed, search-2 is created and search-1 moves to the history") + f.finish("search-1", "Succeeded") f.reconcile() request = f.request() if request.Status.ObservedGeneration != 2 { @@ -214,6 +267,76 @@ func TestV1beta2RequestControllerWaitsForTheActiveRunBeforeStartingTheNext(t *te } } +func TestV1beta2RequestControllerCoalescesIntermediateGenerations(t *testing.T) { + f := newDGDRV1beta2Fixture(t, dgdrV1beta2Request(true)) + f.reconcile() // search-1 is active + + f.bumpGeneration(2) + f.reconcile() + f.bumpGeneration(3) + f.reconcile() + + // Completing search-1 admits the current level, generation 3. + f.finish("search-1", "Succeeded") + f.reconcile() + + // Generation 2 was an intermediate desired state and is never materialized. + names := f.runNames() + if len(names) != 2 { + t.Fatalf("runs = %v, want search-1 and search-3", names) + } + for _, name := range names { + if name != "search-1" && name != "search-3" { + t.Fatalf("runs = %v, want search-1 and search-3 only", names) + } + } +} + +func TestV1beta2RequestControllerIgnoresAPhaseWithoutTheCompletedCondition(t *testing.T) { + f := newDGDRV1beta2Fixture(t, dgdrV1beta2Request(true)) + f.reconcile() + f.bumpGeneration(2) + + f.setPhaseOnly("search-1", "Succeeded") + f.reconcile() + + if names := f.runNames(); len(names) != 1 { + t.Fatalf("runs = %v, want the next Run held back until Completed is published", names) + } + if condition := dgdrV1beta2AcceptedCondition(t, f.request()); condition.Reason != dgdrReasonWaitingForActiveRun { + t.Fatalf("condition = %v, want reason %s", condition, dgdrReasonWaitingForActiveRun) + } +} + +func TestV1beta2RequestControllerReleasesOnCompletedWithoutATerminalPhase(t *testing.T) { + f := newDGDRV1beta2Fixture(t, dgdrV1beta2Request(true)) + f.reconcile() + f.bumpGeneration(2) + + f.setCompletedOnly("search-1") // the phase is still empty + f.reconcile() + + if names := f.runNames(); len(names) != 2 { + t.Fatalf("runs = %v, want search-2 started once Completed is True", names) + } +} + +func TestV1beta2RequestControllerCountsAnUnlabeledOwnedRunAsActive(t *testing.T) { + request := dgdrV1beta2Request(true) + active := dgdrV1beta2OwnedRun(request, 1, false) // correct controller OwnerReference, no labels + f := newDGDRV1beta2Fixture(t, request, active) + f.bumpGeneration(2) + + f.reconcile() + + if names := f.runNames(); len(names) != 1 || names[0] != "search-1" { + t.Fatalf("runs = %v, want no second active Run next to search-1", names) + } + if ref := f.request().Status.ActiveRunRef; ref == nil || ref.Name != "search-1" { + t.Fatalf("activeRunRef = %v, want search-1", ref) + } +} + func TestV1beta2RequestControllerCreatesNoRunWithoutHardware(t *testing.T) { f := newDGDRV1beta2Fixture(t, dgdrV1beta2Request(false)) f.reconcile() @@ -221,7 +344,7 @@ func TestV1beta2RequestControllerCreatesNoRunWithoutHardware(t *testing.T) { t.Fatalf("runs = %v, want none", names) } request := f.request() - condition := dgdrV1beta2ReconciledCondition(t, request) + condition := dgdrV1beta2AcceptedCondition(t, request) if condition.Status != metav1.ConditionFalse || condition.Reason != dgdrReasonInvalidSpec { t.Fatalf("condition = %v, want False/%s", condition, dgdrReasonInvalidSpec) } @@ -244,17 +367,26 @@ func TestV1beta2RequestControllerLeavesForeignCarriersAlone(t *testing.T) { } } +func dgdrV1beta2RunFixtures(request *nvidiacomv1beta2.DynamoGraphDeploymentRequest, count int) []client.Object { + objects := []client.Object{request} + for generation := 1; generation <= count; generation++ { + objects = append(objects, dgdrV1beta2OwnedRun(request, generation, true)) + } + return objects +} + func TestV1beta2RequestControllerRetainsFiveTerminalRuns(t *testing.T) { request := dgdrV1beta2Request(true) request.Generation = 8 - objects := []client.Object{request} - for generation := 1; generation <= 7; generation++ { - objects = append(objects, dgdrV1beta2TerminalRun(request, generation)) - } - f := newDGDRV1beta2Fixture(t, objects...) + f := newDGDRV1beta2Fixture(t, dgdrV1beta2RunFixtures(request, 7)...) f.reconcile() t.Log("The two oldest terminal runs are deleted and the newest five are listed newest first") + assertRetained(t, f) +} + +func assertRetained(t *testing.T, f *dgdrV1beta2Fixture) { + t.Helper() for _, name := range []string{"search-1", "search-2"} { err := f.client.Get(t.Context(), types.NamespacedName{Namespace: "default", Name: name}, &nvidiacomv1beta2.DynamoGraphDeploymentRun{}) if !apierrors.IsNotFound(err) { @@ -273,12 +405,76 @@ func TestV1beta2RequestControllerRetainsFiveTerminalRuns(t *testing.T) { } } -func dgdrV1beta2TerminalRun(request *nvidiacomv1beta2.DynamoGraphDeploymentRequest, generation int) *nvidiacomv1beta2.DynamoGraphDeploymentRun { +func TestV1beta2RequestControllerRetentionIgnoresTheGenerationAnnotation(t *testing.T) { + request := dgdrV1beta2Request(true) + request.Generation = 8 + objects := dgdrV1beta2RunFixtures(request, 7) + for i, object := range objects[1:] { + if i%2 == 0 { + delete(object.GetAnnotations(), dgdrV1beta2GenerationAnnotation) // missing + } else { + object.GetAnnotations()[dgdrV1beta2GenerationAnnotation] = "garbage" // malformed + } + } + f := newDGDRV1beta2Fixture(t, objects...) + f.reconcile() + + assertRetained(t, f) +} + +func TestV1beta2RequestControllerRejectsAControlledRunWithoutAGenerationInItsName(t *testing.T) { + request := dgdrV1beta2Request(true) + odd := dgdrV1beta2OwnedRun(request, 1, true) + odd.Name = "unrelated" + f := newDGDRV1beta2Fixture(t, request, odd) + + if err := f.tryReconcile(); err == nil || !strings.Contains(err.Error(), "unrelated") { + t.Fatalf("err = %v, want an error naming the run", err) + } + if names := f.runNames(); len(names) != 1 { + t.Fatalf("runs = %v, want the odd run left alone and none created", names) + } +} + +func TestV1beta2RequestControllerPublishesStatusBeforeDeletingRetiredRuns(t *testing.T) { + request := dgdrV1beta2Request(true) + request.Generation = 8 + failPatch := true + f := newDGDRV1beta2FixtureWithInterceptor(t, interceptor.Funcs{ + SubResourcePatch: func( + ctx context.Context, c client.Client, subResource string, obj client.Object, + patch client.Patch, opts ...client.SubResourcePatchOption, + ) error { + if failPatch { + return errors.New("injected status patch failure") + } + return c.SubResource(subResource).Patch(ctx, obj, patch, opts...) + }, + }, dgdrV1beta2RunFixtures(request, 7)...) + + if err := f.tryReconcile(); err == nil { + t.Fatal("reconcile succeeded although the status patch failed") + } + // search-8 was created; the two oldest terminal Runs must still exist because the status did not + // move on. + if names := f.runNames(); len(names) != 8 { + t.Fatalf("runs = %v, want all eight: nothing may be deleted before the status is published", names) + } + + failPatch = false + f.reconcile() + assertRetained(t, f) +} + +func dgdrV1beta2OwnedRun( + request *nvidiacomv1beta2.DynamoGraphDeploymentRequest, + generation int, + terminal bool, +) *nvidiacomv1beta2.DynamoGraphDeploymentRun { run := &nvidiacomv1beta2.DynamoGraphDeploymentRun{ ObjectMeta: metav1.ObjectMeta{ - Name: fmt.Sprintf("search-%d", generation), - Namespace: "default", - Labels: map[string]string{dgdrV1beta2RequestLabel: string(request.UID)}, + Name: fmt.Sprintf("%s-%d", request.Name, generation), + Namespace: request.Namespace, Annotations: map[string]string{dgdrV1beta2GenerationAnnotation: fmt.Sprint(generation)}, OwnerReferences: []metav1.OwnerReference{{ APIVersion: nvidiacomv1beta2.GroupVersion.String(), @@ -289,7 +485,10 @@ func dgdrV1beta2TerminalRun(request *nvidiacomv1beta2.DynamoGraphDeploymentReque }}, }, } - run.Status.Phase = nvidiacomv1beta2.DGDRRunPhase(dgdrRunPhaseSucceeded) + if terminal { + run.Status.Phase = nvidiacomv1beta2.DGDRRunPhase("Succeeded") + setCompletedForTest(&run.Status) + } return run } @@ -298,23 +497,6 @@ func boolPointerForTest(value bool) *bool { return &value } // apiextensionsJSONForTest is a non-empty v1beta1 carrier payload. var apiextensionsJSONForTest = apiextensionsv1.JSON{Raw: []byte(`{"model":"carrier"}`)} -func TestV1beta2RequestControllerLabelsRunsWithTheRequestUID(t *testing.T) { - request := dgdrV1beta2Request(true) - request.Name = strings.Repeat("a", 100) - f := newDGDRV1beta2Fixture(t, request) - run, err := f.reconciler.buildRun(request) - if err != nil { - t.Fatalf("build run: %v", err) - } - value := run.Labels[dgdrV1beta2RequestLabel] - if value != string(request.UID) { - t.Fatalf("label = %q, want the request UID", value) - } - if len(value) > 63 { - t.Fatalf("label value has %d characters; a label value is limited to 63", len(value)) - } -} - func TestV1beta2RequestControllerRejectsANameThatCannotBeDerived(t *testing.T) { request := dgdrV1beta2Request(true) request.Name = strings.Repeat("a", 252) @@ -329,7 +511,6 @@ func TestV1beta2RequestControllerDoesNotAdoptARunOfAnotherOwner(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: "search-1", Namespace: "default", - Labels: map[string]string{dgdrV1beta2RequestLabel: "old-uid"}, OwnerReferences: []metav1.OwnerReference{{ APIVersion: nvidiacomv1beta2.GroupVersion.String(), Kind: "DynamoGraphDeploymentRequest", @@ -351,7 +532,7 @@ func TestV1beta2RequestControllerDoesNotAdoptARunOfAnotherOwner(t *testing.T) { t.Fatal("no retry was scheduled for the Run name held by another owner") } request := f.request() - if condition := dgdrV1beta2ReconciledCondition(t, request); condition.Reason != dgdrReasonRunNameConflict { + if condition := dgdrV1beta2AcceptedCondition(t, request); condition.Reason != dgdrReasonRunNameConflict { t.Fatalf("condition = %v, want reason %s", condition, dgdrReasonRunNameConflict) } if request.Status.ActiveRunRef != nil { @@ -361,3 +542,32 @@ func TestV1beta2RequestControllerDoesNotAdoptARunOfAnotherOwner(t *testing.T) { t.Fatalf("observedGeneration = %d, want 0", request.Status.ObservedGeneration) } } + +func TestV1beta2RequestControllerKeepsNullDeletedKeysOutOfTheRunInput(t *testing.T) { + request := dgdrV1beta2Request(true) + // The Request removes a typed value (workload isl) and a derived default (request_count) with null. + const spec = `{"model":{"name":"Qwen/Qwen3-0.6B"},"hardware":{"gpu":{"sku":"h100_sxm","budget":4}},` + + `"workload":{"static":{"isl":1024}},` + + `"aisimulate":{"image":"example.com/aisimulate:latest","versionOverride":"0.13.0",` + + `"parameters":{"workload":{"isl":null,"request_count":null}}}}` + request.Spec = nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec{} + if err := json.Unmarshal([]byte(spec), &request.Spec); err != nil { + t.Fatalf("decode spec: %v", err) + } + f := newDGDRV1beta2Fixture(t, request) + + run, err := f.reconciler.buildRun(request) + if err != nil { + t.Fatalf("build run: %v", err) + } + input, err := aisimulate.InputForRun(run) + if err != nil { + t.Fatalf("input of the run: %v", err) + } + workload, _ := input["workload"].(map[string]any) + for _, key := range []string{"isl", "request_count"} { + if value, found := workload[key]; found { + t.Errorf("workload.%s = %v in the Run input, want it absent: the deletion must stay", key, value) + } + } +} From bb0080e578a29fee2f086a3ea4d5c1a5e79f606c Mon Sep 17 00:00:00 2001 From: devivasudevan <49675305+devivasudevan@users.noreply.github.com> Date: Thu, 22 Oct 2026 17:04:29 +0200 Subject: [PATCH 28/40] fix(operator): derive a valid Run name for every Request name Keep - when it fits; otherwise truncate the Request name and add a stable hash so long names stay valid and distinct. Validate a Run's generation by recomputing its expected name. Register the v1beta2 scheme once in the manager. Signed-off-by: devivasudevan <49675305+devivasudevan@users.noreply.github.com> --- deploy/operator/cmd/main.go | 2 - ...aphdeploymentrequest_v1beta2_controller.go | 42 ++++++++++----- ...ploymentrequest_v1beta2_controller_test.go | 54 +++++++++++++++++-- 3 files changed, 81 insertions(+), 17 deletions(-) diff --git a/deploy/operator/cmd/main.go b/deploy/operator/cmd/main.go index 27fe6de2b945..7c1b789434e1 100644 --- a/deploy/operator/cmd/main.go +++ b/deploy/operator/cmd/main.go @@ -121,8 +121,6 @@ func initCRDSchemes() { utilruntime.Must(nvidiacomv1beta1.AddToScheme(crdScheme)) utilruntime.Must(nvidiacomv1beta2.AddToScheme(crdScheme)) - utilruntime.Must(nvidiacomv1beta2.AddToScheme(crdScheme)) - utilruntime.Must(lwsscheme.AddToScheme(crdScheme)) utilruntime.Must(volcanoscheme.AddToScheme(crdScheme)) diff --git a/deploy/operator/internal/controller/dynamographdeploymentrequest_v1beta2_controller.go b/deploy/operator/internal/controller/dynamographdeploymentrequest_v1beta2_controller.go index 5fa7f6e75133..84df14ccf887 100644 --- a/deploy/operator/internal/controller/dynamographdeploymentrequest_v1beta2_controller.go +++ b/deploy/operator/internal/controller/dynamographdeploymentrequest_v1beta2_controller.go @@ -19,6 +19,8 @@ package controller import ( "context" + "crypto/sha256" + "encoding/hex" "encoding/json" "errors" "fmt" @@ -205,10 +207,6 @@ func (r *DynamoGraphDeploymentRequestV1beta2Reconciler) Reconcile( func (r *DynamoGraphDeploymentRequestV1beta2Reconciler) buildRun( request *nvidiacomv1beta2.DynamoGraphDeploymentRequest, ) (*nvidiacomv1beta2.DynamoGraphDeploymentRun, error) { - if name := dgdrV1beta2RunName(request); len(name) > dgdrV1beta2MaxRunNameLength { - return nil, fmt.Errorf("the Request name is too long: the Run name %q would exceed %d characters", - name, dgdrV1beta2MaxRunNameLength) - } spec := &request.Spec if spec.AISimulate == nil || spec.AISimulate.Image == "" { return nil, errors.New("spec.aisimulate.image is required") @@ -301,7 +299,23 @@ func dgdrRunControllerUIDs(obj client.Object) []string { // dgdrV1beta2RunName names the Run of one Request generation. A metadata-only update keeps the // generation, so it never creates a second Run. func dgdrV1beta2RunName(request *nvidiacomv1beta2.DynamoGraphDeploymentRequest) string { - return fmt.Sprintf("%s-%d", request.Name, request.Generation) + return dgdrV1beta2RunNameFor(request.Name, request.Generation) +} + +// dgdrV1beta2RunNameFor is - whenever that fits in an object name. A longer +// name keeps the generation suffix and a stable hash of the full Request name, and truncates only the +// Request-name part, so every valid Request name has a valid Run name and two long names that share a +// prefix still get different Run names. +func dgdrV1beta2RunNameFor(requestName string, generation int64) string { + suffix := "-" + strconv.FormatInt(generation, 10) + if len(requestName)+len(suffix) <= dgdrV1beta2MaxRunNameLength { + return requestName + suffix + } + sum := sha256.Sum256([]byte(requestName)) + hash := hex.EncodeToString(sum[:6]) + maxPrefix := dgdrV1beta2MaxRunNameLength - len(suffix) - 1 - len(hash) + prefix := strings.TrimRight(requestName[:maxPrefix], "-.") + return prefix + "-" + hash + suffix } func dgdrV1beta2Condition( @@ -324,18 +338,22 @@ func dgdrV1beta2IsTerminal(run *nvidiacomv1beta2.DynamoGraphDeploymentRun) bool return meta.IsStatusConditionTrue(run.Status.Conditions, nvidiacomv1beta2.DGDRRunConditionCompleted) } -// dgdrV1beta2RunGeneration reads the Request generation from the immutable Run name -// (-). A Run that does not follow it is an error, never generation zero. +// dgdrV1beta2RunGeneration reads the Request generation from the immutable Run name. The name is +// accepted only if it is exactly the name this Request derives for that generation; anything else is an +// error, never generation zero. func dgdrV1beta2RunGeneration(requestName string, run *nvidiacomv1beta2.DynamoGraphDeploymentRun) (int64, error) { - prefix := requestName + "-" - value, ok := strings.CutPrefix(run.Name, prefix) - if !ok { - return 0, fmt.Errorf("run %q does not use the Request prefix %q", run.Name, prefix) + index := strings.LastIndex(run.Name, "-") + if index < 0 { + return 0, fmt.Errorf("run %q has no Request generation suffix", run.Name) } - generation, err := strconv.ParseInt(value, 10, 64) + generation, err := strconv.ParseInt(run.Name[index+1:], 10, 64) if err != nil { return 0, fmt.Errorf("run %q has no valid Request generation: %w", run.Name, err) } + if expected := dgdrV1beta2RunNameFor(requestName, generation); run.Name != expected { + return 0, fmt.Errorf("run %q is not the Run name of generation %d of Request %q (expected %q)", + run.Name, generation, requestName, expected) + } return generation, nil } diff --git a/deploy/operator/internal/controller/dynamographdeploymentrequest_v1beta2_controller_test.go b/deploy/operator/internal/controller/dynamographdeploymentrequest_v1beta2_controller_test.go index 5c6e1ee0a9d6..953eb8e1fa02 100644 --- a/deploy/operator/internal/controller/dynamographdeploymentrequest_v1beta2_controller_test.go +++ b/deploy/operator/internal/controller/dynamographdeploymentrequest_v1beta2_controller_test.go @@ -497,12 +497,60 @@ func boolPointerForTest(value bool) *bool { return &value } // apiextensionsJSONForTest is a non-empty v1beta1 carrier payload. var apiextensionsJSONForTest = apiextensionsv1.JSON{Raw: []byte(`{"model":"carrier"}`)} -func TestV1beta2RequestControllerRejectsANameThatCannotBeDerived(t *testing.T) { +func TestV1beta2RequestControllerKeepsTheReadableRunNameWhenItFits(t *testing.T) { + request := dgdrV1beta2Request(true) + request.Name = strings.Repeat("a", 250) + request.Generation = 3 + if got, want := dgdrV1beta2RunName(request), request.Name+"-3"; got != want { + t.Fatalf("run name = %q, want %q", got, want) + } +} + +func TestV1beta2RequestControllerDerivesDistinctValidNamesForOverlengthRequests(t *testing.T) { + a := dgdrV1beta2Request(true) + a.Name = strings.Repeat("a", 252) + b := dgdrV1beta2Request(true) + b.Name = strings.Repeat("a", 251) + "b" // the same retained prefix as a + nameA, nameB := dgdrV1beta2RunName(a), dgdrV1beta2RunName(b) + for _, name := range []string{nameA, nameB} { + if len(name) > dgdrV1beta2MaxRunNameLength || !strings.HasSuffix(name, "-1") { + t.Fatalf("run name %q (%d characters) is not a valid name with the generation suffix", name, len(name)) + } + } + if nameA == nameB { + t.Fatalf("two different Request names produced the same Run name %q", nameA) + } + + t.Log("The name maps back to its generation for its own Request only") + run := &nvidiacomv1beta2.DynamoGraphDeploymentRun{ObjectMeta: metav1.ObjectMeta{Name: nameA}} + if generation, err := dgdrV1beta2RunGeneration(a.Name, run); err != nil || generation != 1 { + t.Fatalf("generation = %d, err = %v, want 1", generation, err) + } + if _, err := dgdrV1beta2RunGeneration(b.Name, run); err == nil { + t.Fatal("a Run name derived from another Request was accepted") + } +} + +func TestV1beta2RequestControllerReconcilesAnOverlengthRequestToOneRun(t *testing.T) { request := dgdrV1beta2Request(true) request.Name = strings.Repeat("a", 252) f := newDGDRV1beta2Fixture(t, request) - if _, err := f.reconciler.buildRun(request); err == nil || !strings.Contains(err.Error(), "too long") { - t.Fatalf("err = %v, want a name-too-long error", err) + key := types.NamespacedName{Namespace: request.Namespace, Name: request.Name} + for range 3 { + if _, err := f.reconciler.Reconcile(t.Context(), ctrl.Request{NamespacedName: key}); err != nil { + t.Fatalf("reconcile: %v", err) + } + } + want := dgdrV1beta2RunName(request) + if names := f.runNames(); len(names) != 1 || names[0] != want { + t.Fatalf("runs = %v, want only %q", names, want) + } + live := &nvidiacomv1beta2.DynamoGraphDeploymentRequest{} + if err := f.client.Get(t.Context(), key, live); err != nil { + t.Fatalf("get request: %v", err) + } + if live.Status.ActiveRunRef == nil || live.Status.ActiveRunRef.Name != want || live.Status.ObservedGeneration != 1 { + t.Fatalf("status = %+v, want the overlength Run active for generation 1", live.Status) } } From 33f6b7bd7a930a53712f43a74228e688d5f80a66 Mon Sep 17 00:00:00 2001 From: devivasudevan <49675305+devivasudevan@users.noreply.github.com> Date: Fri, 23 Oct 2026 09:42:23 +0200 Subject: [PATCH 29/40] fix(operator): keep spec.aisimulate.parameters opaque in v1beta2 admission Admission validates the typed spec and the image/version selection only. Drop the parameters object check and the effective-hardware validation; hardware discovery reads spec.hardware alone. A Request whose discovery fails is admitted and reported as Accepted=False/InvalidSpec by the Request controller. Signed-off-by: devivasudevan <49675305+devivasudevan@users.noreply.github.com> --- .../internal/controller/aisimulate/doc.go | 8 +-- .../dynamographdeploymentrequest_v1beta2.go | 52 ++----------------- ...namographdeploymentrequest_v1beta2_test.go | 23 ++++---- ...ploymentrequest_validation_envtest_test.go | 34 ++++-------- 4 files changed, 31 insertions(+), 86 deletions(-) diff --git a/deploy/operator/internal/controller/aisimulate/doc.go b/deploy/operator/internal/controller/aisimulate/doc.go index b4510da1fc82..a15cb22952e7 100644 --- a/deploy/operator/internal/controller/aisimulate/doc.go +++ b/deploy/operator/internal/controller/aisimulate/doc.go @@ -18,10 +18,10 @@ // Package aisimulate lowers a v1beta2 DynamoGraphDeploymentRequest spec into the native // AISimulate input that a Run executes. // -// Lowering sits below the Request reconciler. Admission calls Parameters to check that -// spec.aisimulate.parameters is an object, and Lower to check the effective input, for -// example that it names the hardware. The reconciler freezes the output of Lower into the Run, -// so admission and the Run see the same bytes. +// Lowering sits below the Request reconciler. Admission does not call it: spec.aisimulate.parameters +// is opaque there, and the validation of the lowered input belongs to the schema of the AISimulate +// release the Request selects. The reconciler lowers the Request once and freezes the output into the +// Run, which executes that object as it is. // // Lower works in three steps. The typed spec (model, backends, hardware, workload, objective) // is first translated into the native input, the "underlay". The derived defaults that no API diff --git a/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_v1beta2.go b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_v1beta2.go index 9ac68b3b4491..ad26540ca670 100644 --- a/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_v1beta2.go +++ b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_v1beta2.go @@ -24,13 +24,9 @@ import ( nvidiacomv1beta2 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta2" "github.com/ai-dynamo/dynamo/deploy/operator/internal/controller/aisimulate" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/util/validation/field" ) -// parametersPath is where the native AISimulate input lives in a v1beta2 Request. -var parametersPath = field.NewPath("spec", "aisimulate", "parameters") - // validateDynamoGraphDeploymentRequestEnvelopeV1beta2 validates the v1beta2 spec carried in // spec.v1beta2 of a v1beta1 object. raw is that field and fldPath is its path; a spec that // does not decode is reported there. Errors about the decoded spec use the v1beta2 spec's own @@ -48,18 +44,11 @@ func validateDynamoGraphDeploymentRequestEnvelopeV1beta2( } // validateDynamoGraphDeploymentRequestV1beta2 validates a defaulted v1beta2 Request spec: the -// typed fields, that spec.aisimulate.parameters is an object, and that the effective AISimulate -// input names the hardware. The rest of the parameters content is not validated here; the -// AISimulate input schema owns it. Update calls this only when the spec changed, so objects -// stored before this check keep working until they are edited. +// typed fields and the image and version selection. spec.aisimulate.parameters is opaque here: the +// CRD types it as an object, and the validation of its content belongs to the schema of the +// AISimulate release the Request selects. func validateDynamoGraphDeploymentRequestV1beta2(spec *nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec) field.ErrorList { - errs := validateDynamoGraphDeploymentRequestSpecV1beta2(spec, field.NewPath("spec")) - if _, err := aisimulate.Parameters(spec); err != nil { - errs = append(errs, field.Invalid(parametersPath, "", err.Error())) - } else { - errs = append(errs, validateEffectiveHardwareV1beta2(spec)...) - } - return errs + return validateDynamoGraphDeploymentRequestSpecV1beta2(spec, field.NewPath("spec")) } // validateDynamoGraphDeploymentRequestSpecV1beta2 validates spec. spec and fldPath must not be nil. @@ -128,36 +117,3 @@ func validateAISimulateSpecV1beta2(spec *nvidiacomv1beta2.AISimulateSpec, fldPat } return nil } - -// validateEffectiveHardwareV1beta2 requires the AISimulate input that a Run would execute to name -// the GPU SKU and budget. The input is the typed spec lowered and then patched by -// spec.aisimulate.parameters, so hardware given only in the parameters satisfies it, and a -// parameters null that deletes discovered hardware does not. Errors are reported at the typed -// spec.hardware.gpu fields, which is where the user fixes them. -func validateEffectiveHardwareV1beta2(spec *nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec) field.ErrorList { - input, err := aisimulate.Lower(spec) - if err != nil { - return field.ErrorList{field.Invalid(parametersPath, "", err.Error())} - } - gpuPath := field.NewPath("spec", "hardware", "gpu") - var errs field.ErrorList - if !effectiveValuePresent(input, aisimulate.SearchSpaceHardwareSKU) { - errs = append(errs, field.Required(gpuPath.Child("sku"), - "the effective AISimulate input has no hardware SKU: set it, enable GPU discovery, or set spec.aisimulate.parameters.search_space.hardware_sku")) - } - if !effectiveValuePresent(input, aisimulate.SearchSpaceGPUBudget) { - errs = append(errs, field.Required(gpuPath.Child("budget"), - "the effective AISimulate input has no GPU budget: set it, enable GPU discovery, or set spec.aisimulate.parameters.search_space.gpu_budget")) - } - return errs -} - -// effectiveValuePresent reports whether input holds a non-null, non-empty value at path. -func effectiveValuePresent(input map[string]any, path []string) bool { - value, found, err := unstructured.NestedFieldNoCopy(input, path...) - if err != nil || !found || value == nil { - return false - } - text, isString := value.(string) - return !isString || text != "" -} diff --git a/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_v1beta2_test.go b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_v1beta2_test.go index e008602955b5..e0120bcb0827 100644 --- a/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_v1beta2_test.go +++ b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_v1beta2_test.go @@ -117,16 +117,22 @@ func TestValidateV1beta2TypedStage(t *testing.T) { spec.Workload.Static.KVLoadRatio = nvidiacomv1beta2.KVLoadRatio{0.9, 0.1} v1beta2RequireError(t, validateDynamoGraphDeploymentRequestV1beta2(spec), "spec.workload.static.kvLoadRatio") }) - t.Run("typed and parameter errors aggregate", func(t *testing.T) { - spec := v1beta2WithParameters(v1beta2DefaultedSpec(), `[1]`) + t.Run("typed errors aggregate", func(t *testing.T) { + spec := v1beta2DefaultedSpec() spec.Model.Name = "" + spec.Backends = []nvidiacomv1beta2.Backend{nvidiacomv1beta2.BackendVLLM, nvidiacomv1beta2.BackendVLLM} errs := validateDynamoGraphDeploymentRequestV1beta2(spec) v1beta2RequireError(t, errs, "spec.model.name") - v1beta2RequireError(t, errs, "spec.aisimulate.parameters") + v1beta2RequireError(t, errs, "spec.backends[1]") }) - t.Run("parameter content is not validated here", func(t *testing.T) { - v1beta2RequireNoError(t, validateDynamoGraphDeploymentRequestV1beta2( - v1beta2WithParameters(v1beta2DefaultedSpec(), `{"workload":{"isl":0},"goal":{"target":"nope"}}`))) + t.Run("parameters are opaque to admission", func(t *testing.T) { + for _, parameters := range []string{ + `{"workload":{"isl":0},"goal":{"target":"nope"}}`, + `[1]`, // the CRD types parameters as an object; admission does not interpret it + } { + v1beta2RequireNoError(t, validateDynamoGraphDeploymentRequestV1beta2( + v1beta2WithParameters(v1beta2DefaultedSpec(), parameters))) + } }) } @@ -154,11 +160,6 @@ func TestValidateV1beta2AISimulateImage(t *testing.T) { }) } -func TestValidateV1beta2MalformedParameters(t *testing.T) { - errs := validateDynamoGraphDeploymentRequestV1beta2(v1beta2WithParameters(v1beta2DefaultedSpec(), `[1]`)) - v1beta2RequireError(t, errs, "spec.aisimulate.parameters: spec.aisimulate.parameters: must be a JSON object") -} - func v1beta2Envelope(raw string) *apiextensionsv1.JSON { return &apiextensionsv1.JSON{Raw: []byte(raw)} } diff --git a/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_validation_envtest_test.go b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_validation_envtest_test.go index 339ccde92aa3..c8583be88741 100644 --- a/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_validation_envtest_test.go +++ b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_validation_envtest_test.go @@ -575,17 +575,16 @@ func TestDynamoGraphDeploymentRequestValidator_Validate(t *testing.T) { }, }, { - name: "v1beta2 failed discovery is rejected", + name: "v1beta2 failed discovery leaves the hardware unresolved and is admitted", request: v1beta2DGDRForAdmission(nil, nil), gpuDiscovery: true, discoveryErr: errors.New("no GPUs found"), - wantWebhook: []string{ - v1beta2SKURequired, - v1beta2BudgetRequired, + check: func(t *testing.T, result *unstructured.Unstructured) { + v1beta2Absent(t, result, "spec", "hardware") }, }, { - name: "v1beta2 failed discovery permits hardware given in the parameters", + name: "v1beta2 parameters are stored untouched and never decide hardware defaulting", request: v1beta2DGDRForAdmission(func(spec map[string]any) { setV1beta2SearchSpace(spec, map[string]any{"hardware_sku": "h100_sxm", "gpu_budget": int64(4)}) }, nil), @@ -599,22 +598,17 @@ func TestDynamoGraphDeploymentRequestValidator_Validate(t *testing.T) { }, }, { - name: "v1beta2 failed discovery with only one effective hardware key is rejected at the missing one", - request: v1beta2DGDRForAdmission(func(spec map[string]any) { - setV1beta2SearchSpace(spec, map[string]any{"hardware_sku": "h100_sxm"}) - }, nil), - gpuDiscovery: true, - discoveryErr: errors.New("no GPUs found"), - wantWebhook: []string{v1beta2BudgetRequired}, - }, - { - name: "v1beta2 parameters null that deletes discovered hardware is rejected", + name: "v1beta2 parameters null does not suppress hardware discovery", request: v1beta2DGDRForAdmission(func(spec map[string]any) { setV1beta2SearchSpace(spec, map[string]any{"hardware_sku": nil}) }, nil), gpuDiscovery: true, discovered: discoveredH200, - wantWebhook: []string{v1beta2SKURequired}, + check: func(t *testing.T, result *unstructured.Unstructured) { + if got := v1beta2String(t, result, "spec", "hardware", "gpu", "sku"); got == "" { + t.Fatal("hardware.gpu.sku is empty: discovery must read the typed spec only") + } + }, }, { name: "v1beta2 empty workload receives the static model-only default", @@ -1186,13 +1180,7 @@ func setV1beta2SearchSpace(spec map[string]any, searchSpace map[string]any) { aisimulate["parameters"] = map[string]any{"search_space": searchSpace} } -const ( - v1beta2SKURequired = "spec.hardware.gpu.sku: Required value: the effective AISimulate input has no hardware SKU: set it, enable GPU discovery, or set spec.aisimulate.parameters.search_space.hardware_sku" - v1beta2BudgetRequired = "spec.hardware.gpu.budget: Required value: the effective AISimulate input has no GPU budget: set it, enable GPU discovery, or set spec.aisimulate.parameters.search_space.gpu_budget" -) - -// setBeta2AdmissionHardware names the hardware, as admission requires when GPU discovery finds -// none. Tests that are not about hardware use it so their requests are valid. +// setBeta2AdmissionHardware names the hardware, so fixtures do not depend on GPU discovery. func setBeta2AdmissionHardware(request *nvidiacomv1beta2.DynamoGraphDeploymentRequest) { sku, budget := "h100_sxm", int32(4) request.Spec.Hardware = &nvidiacomv1beta2.HardwareSpec{ From b6f0f131fcbe8e35d601ce56af1ca527b1638aab Mon Sep 17 00:00:00 2001 From: devivasudevan <49675305+devivasudevan@users.noreply.github.com> Date: Fri, 23 Oct 2026 10:14:10 +0200 Subject: [PATCH 30/40] refactor(operator): make admission the only owner of v1beta2 DGDR defaults Remove the +kubebuilder:default markers for workload.static, objective.mode and recommendation.maxCandidates, and the test that compared them with the webhook constants. Default in the order of the Go API types: each parent creates its documented default child and fills its own scalars, and workload.static: {} stays an opt-out. Signed-off-by: devivasudevan <49675305+devivasudevan@users.noreply.github.com> --- .../dynamographdeploymentrequest_types.go | 3 - ...dia.com_dynamographdeploymentrequests.yaml | 8 +- .../nvidia.com_dynamographdeploymentruns.yaml | 8 +- .../dynamographdeploymentrequest_v1beta2.go | 104 +++++++----- ...graphdeploymentrequest_v1beta2_crd_test.go | 92 ---------- ...ploymentrequest_v1beta2_structural_test.go | 157 ++++++++++++++++++ ...namographdeploymentrequest_v1beta2_test.go | 26 ++- 7 files changed, 235 insertions(+), 163 deletions(-) delete mode 100644 deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_crd_test.go create mode 100644 deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_structural_test.go diff --git a/deploy/operator/api/v1beta2/dynamographdeploymentrequest_types.go b/deploy/operator/api/v1beta2/dynamographdeploymentrequest_types.go index e20fc4d782d9..21bd766b4030 100644 --- a/deploy/operator/api/v1beta2/dynamographdeploymentrequest_types.go +++ b/deploy/operator/api/v1beta2/dynamographdeploymentrequest_types.go @@ -114,7 +114,6 @@ type WorkloadSpec struct { // defaulted individually: the default belongs to spec.workload as a whole, so an // explicit workload object keeps exactly the fields it names. // +optional - // +kubebuilder:default={isl: 1024, osl: 128, concurrency: 10} Static *StaticWorkloadSpec `json:"static,omitempty"` } @@ -161,7 +160,6 @@ type ObjectiveSLASpec struct { type ObjectiveSpec struct { // Mode selects scalar optimization or a Pareto search. // +optional - // +kubebuilder:default=Optimize Mode ObjectiveMode `json:"mode,omitempty"` // Metric is the scalar target when mode is Optimize. @@ -212,7 +210,6 @@ type RecommendationSpec struct { // candidates are ordered best first; Pareto candidates have no scalar rank. // This field does not limit internal trials. // +optional - // +kubebuilder:default=1 // +kubebuilder:validation:Minimum=1 // +kubebuilder:validation:Maximum=100 MaxCandidates int32 `json:"maxCandidates,omitempty"` diff --git a/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentrequests.yaml b/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentrequests.yaml index 65b19e735a05..9a8e2621d652 100644 --- a/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentrequests.yaml +++ b/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentrequests.yaml @@ -9594,7 +9594,7 @@ spec: image: description: |- Image selects a Dynamo-compatible AISimulate runner. When omitted, admission - persists the operator's configured image. + persists the configured override or a version-matched Dynamo planner image. minLength: 1 type: string parameters: @@ -9695,7 +9695,6 @@ spec: type: array x-kubernetes-list-type: set mode: - default: Optimize description: Mode selects scalar optimization or a Pareto search. enum: - Optimize @@ -9737,7 +9736,6 @@ spec: description: Recommendation controls bounded projection into DGDC resources. properties: maxCandidates: - default: 1 description: |- MaxCandidates is the maximum number of DGDCs owned by one run. Scalar candidates are ordered best first; Pareto candidates have no scalar rank. @@ -9767,10 +9765,6 @@ spec: KV-load ratio search. properties: static: - default: - concurrency: 10 - isl: 1024 - osl: 128 description: |- Static defines one of AISimulate's synthetic workload shapes. Its fields are not defaulted individually: the default belongs to spec.workload as a whole, so an diff --git a/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentruns.yaml b/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentruns.yaml index 7d7cd9bae305..ad6f33e55d3d 100644 --- a/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentruns.yaml +++ b/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentruns.yaml @@ -66,7 +66,7 @@ spec: image: description: |- Image selects a Dynamo-compatible AISimulate runner. When omitted, admission - persists the operator's configured image. + persists the configured override or a version-matched Dynamo planner image. minLength: 1 type: string parameters: @@ -167,7 +167,6 @@ spec: type: array x-kubernetes-list-type: set mode: - default: Optimize description: Mode selects scalar optimization or a Pareto search. enum: - Optimize @@ -209,7 +208,6 @@ spec: description: Recommendation controls bounded projection into DGDC resources. properties: maxCandidates: - default: 1 description: |- MaxCandidates is the maximum number of DGDCs owned by one run. Scalar candidates are ordered best first; Pareto candidates have no scalar rank. @@ -231,10 +229,6 @@ spec: KV-load ratio search. properties: static: - default: - concurrency: 10 - isl: 1024 - osl: 128 description: |- Static defines one of AISimulate's synthetic workload shapes. Its fields are not defaulted individually: the default belongs to spec.workload as a whole, so an diff --git a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2.go b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2.go index ae9014d1e006..cad0e0caf3f2 100644 --- a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2.go +++ b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2.go @@ -27,14 +27,15 @@ import ( nvidiacomv1beta2 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta2" admissionv1 "k8s.io/api/admission/v1" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + "k8s.io/utils/ptr" ) // defaultMetricV1beta2 is the scalar target an Optimize objective receives when it names none. const defaultMetricV1beta2 = nvidiacomv1beta2.ObjectiveMetric("Throughput") -// Nested defaults of the parents admission creates. They must equal the +kubebuilder:default -// markers in api/v1beta2: the API server defaults a mutated object in the webhook's version -// (v1beta1, whose v1beta2 carrier is opaque), so the v1beta2 schema never fills them in. +// Defaults of the objects admission creates. Admission is their only owner: the API server +// defaults a mutated object in the webhook's version (v1beta1, whose v1beta2 carrier is +// opaque), so the v1beta2 schema carries no default markers for them. const ( defaultStaticISLV1beta2 int32 = 1024 defaultStaticOSLV1beta2 int32 = 128 @@ -42,20 +43,18 @@ const ( defaultMaxCandidatesV1beta2 int32 = 1 ) -func defaultPtrV1beta2[T any](v T) *T { return &v } - // This file defaults the v1beta2 shape of a DynamoGraphDeploymentRequest. It is separate // from the canonical defaulting because the v1beta1 hub carries a v1beta2 Request as an // opaque spec.v1beta2 envelope: conversion cannot apply field-level defaults to it, so the // handler decodes the envelope, runs this traversal and encodes the result. Names carry the // API version because the canonical names belong to v1beta1. // -// The CRD schema owns the unconditional defaults: backends, objective.mode, -// recommendation.maxCandidates and workload.static as a whole (workload.static: {} opts -// out). objective.sla has no default: an omitted SLA stays absent. Admission owns what depends on -// other fields or on the environment: the scalar metric, the cluster hardware and the -// AISimulate image. spec.aisimulate.parameters never changes a default. It overrides the -// defaulted typed values later, when the input is lowered. +// Admission owns every v1beta2 default, and each default has this one owner: backends, +// objective.mode, the scalar metric, recommendation.maxCandidates, workload.static, the +// cluster hardware and the AISimulate image. workload.static: {} opts out of the static +// workload. objective.sla has no default: an omitted SLA stays absent. +// spec.aisimulate.parameters never changes a default. It overrides the defaulted typed +// values later, when the input is lowered. // defaultDynamoGraphDeploymentRequestV1beta2 defaults the v1beta2 spec that dgdr carries in // spec.v1beta2. The environment is read once, before any mutation, and only on CREATE: @@ -105,46 +104,35 @@ type hardwareDefaultsV1beta2 struct { budget int32 } -func (h hardwareDefaultsV1beta2) empty() bool { return h.sku == "" && h.budget == 0 } - // defaultDynamoGraphDeploymentRequestSpecV1beta2 defaults spec in place. It applies on -// CREATE and UPDATE and is idempotent; opts carries no hardware on UPDATE. spec must not be nil. +// CREATE and UPDATE and is idempotent; opts carries no hardware on UPDATE. spec must not be +// nil. Explicit values, including empty objects, are kept. Children are visited in the +// declaration order of the API type. func defaultDynamoGraphDeploymentRequestSpecV1beta2( spec *nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec, opts dynamoGraphDeploymentRequestV1beta2DefaultOptions, ) { - // Structural defaults live here, not in the CRD schema: a schema default would also be - // applied to an object carrying spec.v1beta1 and put native fields next to the carrier, - // which conversion rejects. This function only runs for native requests. Explicit values, - // including empty objects, are kept. A parent the client sent already received its nested - // schema defaults at decode; a parent created here gets them explicitly. if spec.Backends == nil { spec.Backends = []nvidiacomv1beta2.Backend{"vllm", "sglang", "trtllm"} } - if spec.Workload == nil { - spec.Workload = &nvidiacomv1beta2.WorkloadSpec{Static: &nvidiacomv1beta2.StaticWorkloadSpec{ - ISL: defaultPtrV1beta2(defaultStaticISLV1beta2), - OSL: defaultPtrV1beta2(defaultStaticOSLV1beta2), - Concurrency: defaultPtrV1beta2(defaultStaticConcurrencyV1beta2), - }} - } - if spec.Objective == nil { - spec.Objective = &nvidiacomv1beta2.ObjectiveSpec{Mode: nvidiacomv1beta2.ObjectiveModeOptimize} - } - if spec.Recommendation == nil { - spec.Recommendation = &nvidiacomv1beta2.RecommendationSpec{MaxCandidates: defaultMaxCandidatesV1beta2} - } - if spec.Hardware == nil && !opts.hardware.empty() { + // Hardware has no static default: it exists only when the cluster supplied some. + if spec.Hardware == nil && opts.hardware != (hardwareDefaultsV1beta2{}) { spec.Hardware = &nvidiacomv1beta2.HardwareSpec{} } if spec.Hardware != nil { defaultHardwareSpecV1beta2(spec.Hardware, opts.hardware) } - if spec.Objective != nil { - defaultObjectiveSpecV1beta2(spec.Objective) + if spec.Workload == nil { + spec.Workload = &nvidiacomv1beta2.WorkloadSpec{} + } + defaultWorkloadSpecV1beta2(spec.Workload) + + if spec.Objective == nil { + spec.Objective = &nvidiacomv1beta2.ObjectiveSpec{} } + defaultObjectiveSpecV1beta2(spec.Objective) // The operator image is persisted with the Request, so spec.aisimulate is created to // carry it. @@ -154,10 +142,15 @@ func defaultDynamoGraphDeploymentRequestSpecV1beta2( if spec.AISimulate != nil { defaultAISimulateSpecV1beta2(spec.AISimulate, opts.aisimulateImage) } + + if spec.Recommendation == nil { + spec.Recommendation = &nvidiacomv1beta2.RecommendationSpec{} + } + defaultRecommendationSpecV1beta2(spec.Recommendation) } func defaultHardwareSpecV1beta2(hardware *nvidiacomv1beta2.HardwareSpec, defaults hardwareDefaultsV1beta2) { - if hardware.GPU == nil && !defaults.empty() { + if hardware.GPU == nil && defaults != (hardwareDefaultsV1beta2{}) { hardware.GPU = &nvidiacomv1beta2.GPUHardwareSpec{} } if hardware.GPU != nil { @@ -169,23 +162,38 @@ func defaultHardwareSpecV1beta2(hardware *nvidiacomv1beta2.HardwareSpec, default // SKU and let admission size the budget. func defaultGPUHardwareSpecV1beta2(gpu *nvidiacomv1beta2.GPUHardwareSpec, defaults hardwareDefaultsV1beta2) { if gpu.SKU == nil && defaults.sku != "" { - sku := defaults.sku - gpu.SKU = &sku + gpu.SKU = ptr.To(defaults.sku) } if gpu.Budget == nil && defaults.budget > 0 { - budget := defaults.budget - gpu.Budget = &budget + gpu.Budget = ptr.To(defaults.budget) + } +} + +func defaultWorkloadSpecV1beta2(workload *nvidiacomv1beta2.WorkloadSpec) { + // An explicit static: {} is the opt-out, so the static defaults apply only to the object + // created here. + if workload.Static == nil { + workload.Static = &nvidiacomv1beta2.StaticWorkloadSpec{} + defaultStaticWorkloadSpecV1beta2(workload.Static) } } -// defaultObjectiveSpecV1beta2 gives an Optimize objective without a metric the default -// scalar target. It reads the typed objective alone: a goal.target in +func defaultStaticWorkloadSpecV1beta2(static *nvidiacomv1beta2.StaticWorkloadSpec) { + static.ISL = ptr.To(defaultStaticISLV1beta2) + static.OSL = ptr.To(defaultStaticOSLV1beta2) + static.Concurrency = ptr.To(defaultStaticConcurrencyV1beta2) +} + +// defaultObjectiveSpecV1beta2 defaults the mode and gives an Optimize objective without a +// metric the default scalar target. It reads the typed objective alone: a goal.target in // spec.aisimulate.parameters overrides the completed typed value when the input is lowered, // so it never suppresses this default. func defaultObjectiveSpecV1beta2(objective *nvidiacomv1beta2.ObjectiveSpec) { - if (objective.Mode == "" || objective.Mode == nvidiacomv1beta2.ObjectiveModeOptimize) && objective.Metric == nil { - metric := defaultMetricV1beta2 - objective.Metric = &metric + if objective.Mode == "" { + objective.Mode = nvidiacomv1beta2.ObjectiveModeOptimize + } + if objective.Mode == nvidiacomv1beta2.ObjectiveModeOptimize && objective.Metric == nil { + objective.Metric = ptr.To(defaultMetricV1beta2) } } @@ -194,3 +202,9 @@ func defaultAISimulateSpecV1beta2(aisimulate *nvidiacomv1beta2.AISimulateSpec, i aisimulate.Image = image } } + +func defaultRecommendationSpecV1beta2(recommendation *nvidiacomv1beta2.RecommendationSpec) { + if recommendation.MaxCandidates == 0 { + recommendation.MaxCandidates = defaultMaxCandidatesV1beta2 + } +} diff --git a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_crd_test.go b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_crd_test.go deleted file mode 100644 index 804761355ae9..000000000000 --- a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_crd_test.go +++ /dev/null @@ -1,92 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - */ - -package defaulting - -import ( - "encoding/json" - "os" - "path/filepath" - "reflect" - "runtime" - "testing" - - apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" - "sigs.k8s.io/yaml" -) - -// v1beta2SpecSchema returns the spec schema of the generated v1beta2 Request CRD. -func v1beta2SpecSchema(t *testing.T) apiextensionsv1.JSONSchemaProps { - t.Helper() - _, thisFile, _, _ := runtime.Caller(0) - path := filepath.Join(filepath.Dir(thisFile), "../../../config/crd/bases", "nvidia.com_dynamographdeploymentrequests.yaml") - raw, err := os.ReadFile(path) - if err != nil { - t.Fatalf("read the generated CRD: %v", err) - } - var crd apiextensionsv1.CustomResourceDefinition - if err := yaml.Unmarshal(raw, &crd); err != nil { - t.Fatalf("decode the generated CRD: %v", err) - } - for _, version := range crd.Spec.Versions { - if version.Name == "v1beta2" { - return version.Schema.OpenAPIV3Schema.Properties["spec"] - } - } - t.Fatal("the generated CRD has no v1beta2 version") - return apiextensionsv1.JSONSchemaProps{} -} - -func schemaDefault(t *testing.T, prop apiextensionsv1.JSONSchemaProps) any { - t.Helper() - if prop.Default == nil { - return nil - } - var value any - if err := json.Unmarshal(prop.Default.Raw, &value); err != nil { - t.Fatalf("decode a schema default: %v", err) - } - return value -} - -// A schema default on a top-level native field is also applied to an object that carries -// spec.v1beta1, which puts native fields next to the carrier and makes conversion reject it. -func TestV1beta2CRDHasNoTopLevelStructuralDefaults(t *testing.T) { - spec := v1beta2SpecSchema(t) - for _, field := range []string{"backends", "workload", "objective", "recommendation"} { - prop, ok := spec.Properties[field] - if !ok { - t.Fatalf("spec.%s is missing from the v1beta2 schema", field) - } - if prop.Default != nil { - t.Errorf("spec.%s has a schema default %s; top-level defaults belong in admission", field, string(prop.Default.Raw)) - } - } -} - -// Admission creates workload, objective and recommendation itself, and the API server then -// defaults the mutated object in the webhook's version, so the nested schema defaults would -// not apply to them. The webhook sets those values explicitly and they must equal the schema. -func TestV1beta2NestedDefaultsMatchTheCRD(t *testing.T) { - spec := v1beta2SpecSchema(t) - - wantStatic := map[string]any{ - "isl": float64(defaultStaticISLV1beta2), - "osl": float64(defaultStaticOSLV1beta2), - "concurrency": float64(defaultStaticConcurrencyV1beta2), - } - got := schemaDefault(t, spec.Properties["workload"].Properties["static"]) - if !reflect.DeepEqual(got, wantStatic) { - t.Errorf("workload.static default = %v, webhook constants = %v", got, wantStatic) - } - - if got := schemaDefault(t, spec.Properties["objective"].Properties["mode"]); got != "Optimize" { - t.Errorf("objective.mode default = %v, webhook sets Optimize", got) - } - - if got := schemaDefault(t, spec.Properties["recommendation"].Properties["maxCandidates"]); got != float64(defaultMaxCandidatesV1beta2) { - t.Errorf("recommendation.maxCandidates default = %v, webhook constant = %d", got, defaultMaxCandidatesV1beta2) - } -} diff --git a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_structural_test.go b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_structural_test.go new file mode 100644 index 000000000000..270e3732a5a1 --- /dev/null +++ b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_structural_test.go @@ -0,0 +1,157 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package defaulting + +import ( + "encoding/json" + "testing" + + nvidiacomv1beta2 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta2" + "k8s.io/utils/ptr" +) + +func structuralTestOptions() dynamoGraphDeploymentRequestV1beta2DefaultOptions { + return dynamoGraphDeploymentRequestV1beta2DefaultOptions{ + hardware: hardwareDefaultsV1beta2{sku: "h200_sxm", budget: 8}, + aisimulateImage: "planner:test", + } +} + +func structuralDefaultedSpec() nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec { + return nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec{ + Backends: []nvidiacomv1beta2.Backend{"vllm", "sglang", "trtllm"}, + Hardware: &nvidiacomv1beta2.HardwareSpec{GPU: &nvidiacomv1beta2.GPUHardwareSpec{ + SKU: ptr.To("h200_sxm"), Budget: ptr.To(int32(8)), + }}, + Workload: &nvidiacomv1beta2.WorkloadSpec{Static: &nvidiacomv1beta2.StaticWorkloadSpec{ + ISL: ptr.To(int32(1024)), OSL: ptr.To(int32(128)), Concurrency: ptr.To(int32(10)), + }}, + Objective: &nvidiacomv1beta2.ObjectiveSpec{ + Mode: nvidiacomv1beta2.ObjectiveModeOptimize, + Metric: ptr.To(nvidiacomv1beta2.ObjectiveMetric("Throughput")), + }, + AISimulate: &nvidiacomv1beta2.AISimulateSpec{Image: "planner:test"}, + Recommendation: &nvidiacomv1beta2.RecommendationSpec{MaxCandidates: 1}, + } +} + +func mustJSON(t *testing.T, v any) string { + t.Helper() + raw, err := json.Marshal(v) + if err != nil { + t.Fatal(err) + } + return string(raw) +} + +func TestStructuralDefaultsV1beta2(t *testing.T) { + tests := []struct { + name string + input nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec + want func() nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec + }{ + { + name: "an empty spec receives every default", + input: nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec{}, + want: structuralDefaultedSpec, + }, + { + name: "empty parents receive their own defaults", + input: nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec{ + Workload: &nvidiacomv1beta2.WorkloadSpec{}, + Objective: &nvidiacomv1beta2.ObjectiveSpec{}, + Recommendation: &nvidiacomv1beta2.RecommendationSpec{}, + }, + want: structuralDefaultedSpec, + }, + { + name: "workload.static: {} is an opt-out and stays empty", + input: nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec{ + Workload: &nvidiacomv1beta2.WorkloadSpec{Static: &nvidiacomv1beta2.StaticWorkloadSpec{}}, + }, + want: func() nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec { + spec := structuralDefaultedSpec() + spec.Workload = &nvidiacomv1beta2.WorkloadSpec{Static: &nvidiacomv1beta2.StaticWorkloadSpec{}} + return spec + }, + }, + { + name: "explicit values are kept", + input: nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec{ + Backends: []nvidiacomv1beta2.Backend{"vllm"}, + Hardware: &nvidiacomv1beta2.HardwareSpec{GPU: &nvidiacomv1beta2.GPUHardwareSpec{ + SKU: ptr.To("h100_sxm"), Budget: ptr.To(int32(4)), + }}, + Workload: &nvidiacomv1beta2.WorkloadSpec{Static: &nvidiacomv1beta2.StaticWorkloadSpec{ + ISL: ptr.To(int32(10)), + }}, + Objective: &nvidiacomv1beta2.ObjectiveSpec{ + Mode: nvidiacomv1beta2.ObjectiveModeOptimize, + Metric: ptr.To(nvidiacomv1beta2.ObjectiveMetric("TTFT")), + }, + AISimulate: &nvidiacomv1beta2.AISimulateSpec{Image: "mine:1"}, + Recommendation: &nvidiacomv1beta2.RecommendationSpec{MaxCandidates: 3}, + }, + want: func() nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec { + return nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec{ + Backends: []nvidiacomv1beta2.Backend{"vllm"}, + Hardware: &nvidiacomv1beta2.HardwareSpec{GPU: &nvidiacomv1beta2.GPUHardwareSpec{ + SKU: ptr.To("h100_sxm"), Budget: ptr.To(int32(4)), + }}, + Workload: &nvidiacomv1beta2.WorkloadSpec{Static: &nvidiacomv1beta2.StaticWorkloadSpec{ + ISL: ptr.To(int32(10)), + }}, + Objective: &nvidiacomv1beta2.ObjectiveSpec{ + Mode: nvidiacomv1beta2.ObjectiveModeOptimize, + Metric: ptr.To(nvidiacomv1beta2.ObjectiveMetric("TTFT")), + }, + AISimulate: &nvidiacomv1beta2.AISimulateSpec{Image: "mine:1"}, + Recommendation: &nvidiacomv1beta2.RecommendationSpec{MaxCandidates: 3}, + } + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + spec := tt.input + defaultDynamoGraphDeploymentRequestSpecV1beta2(&spec, structuralTestOptions()) + want := tt.want() + if got, wantJSON := mustJSON(t, &spec), mustJSON(t, &want); got != wantJSON { + t.Fatalf("defaulted spec mismatch\n got: %s\nwant: %s", got, wantJSON) + } + + // Defaulting is idempotent. + again := spec + defaultDynamoGraphDeploymentRequestSpecV1beta2(&again, structuralTestOptions()) + if got, wantJSON := mustJSON(t, &again), mustJSON(t, &spec); got != wantJSON { + t.Fatalf("second pass changed the spec\n got: %s\nwant: %s", got, wantJSON) + } + }) + } +} + +func TestStructuralDefaultsV1beta2WithoutEnvironment(t *testing.T) { + var spec nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec + defaultDynamoGraphDeploymentRequestSpecV1beta2(&spec, dynamoGraphDeploymentRequestV1beta2DefaultOptions{}) + if spec.Hardware != nil || spec.AISimulate != nil { + t.Fatalf("hardware and aisimulate must stay absent without an environment: %s", mustJSON(t, &spec)) + } + if spec.Workload == nil || spec.Objective == nil || spec.Recommendation == nil || spec.Backends == nil { + t.Fatalf("static defaults must still apply: %s", mustJSON(t, &spec)) + } +} diff --git a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_test.go b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_test.go index decb7fc49b23..55b90fabff49 100644 --- a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_test.go +++ b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_test.go @@ -156,17 +156,25 @@ func TestDefaultSpecV1beta2PreservesExplicitValues(t *testing.T) { }) } -func TestDefaultSpecV1beta2NeverTouchesTheSchemaOwnedValues(t *testing.T) { +func TestDefaultSpecV1beta2KeepsExplicitValues(t *testing.T) { opts := dynamoGraphDeploymentRequestV1beta2DefaultOptions{ hardware: hardwareDefaultsV1beta2{sku: "h200_sxm", budget: 8}, aisimulateImage: "registry.example.com/aisimulate:1.2.0", } - t.Run("explicit empty workload", func(t *testing.T) { + t.Run("an empty workload receives the static default", func(t *testing.T) { spec := v1beta2SchemaDefaulted() spec.Workload = &nvidiacomv1beta2.WorkloadSpec{} defaultDynamoGraphDeploymentRequestSpecV1beta2(spec, opts) - if spec.Workload.Static != nil { - t.Fatal("an explicit workload: {} was filled") + if !hasModelOnlyStatic(spec.Workload) { + t.Fatal("workload: {} must receive the static model-only default") + } + }) + t.Run("an explicit empty static workload is an opt-out", func(t *testing.T) { + spec := v1beta2SchemaDefaulted() + spec.Workload = &nvidiacomv1beta2.WorkloadSpec{Static: &nvidiacomv1beta2.StaticWorkloadSpec{}} + defaultDynamoGraphDeploymentRequestSpecV1beta2(spec, opts) + if spec.Workload.Static.ISL != nil || spec.Workload.Static.OSL != nil || spec.Workload.Static.Concurrency != nil { + t.Fatal("an explicit workload.static: {} was filled") } }) t.Run("omitted workload", func(t *testing.T) { @@ -204,7 +212,7 @@ func TestDefaultObjectiveSpecV1beta2(t *testing.T) { {"optimize without a metric", nvidiacomv1beta2.ObjectiveSpec{Mode: nvidiacomv1beta2.ObjectiveModeOptimize}, "Throughput"}, {"an explicit metric is kept", nvidiacomv1beta2.ObjectiveSpec{Mode: nvidiacomv1beta2.ObjectiveModeOptimize, Metric: v1beta2Ptr(nvidiacomv1beta2.ObjectiveMetric("TTFT"))}, "TTFT"}, {"pareto has no scalar metric", nvidiacomv1beta2.ObjectiveSpec{Mode: nvidiacomv1beta2.ObjectiveModePareto, Metrics: []nvidiacomv1beta2.ObjectiveMetric{"TTFT", "Throughput"}}, ""}, - {"an absent mode is Optimize, which the schema fills after admission", nvidiacomv1beta2.ObjectiveSpec{}, "Throughput"}, + {"an absent mode is Optimize, which admission fills", nvidiacomv1beta2.ObjectiveSpec{}, "Throughput"}, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { @@ -272,14 +280,14 @@ func TestResolveHardwareV1beta2(t *testing.T) { } t.Run("no discoverer", func(t *testing.T) { - if got := (DGDRV1beta2Options{}).resolveHardware(context.Background(), admissionv1.Create, spec(nil, nil)); !got.empty() { + if got := (DGDRV1beta2Options{}).resolveHardware(context.Background(), admissionv1.Create, spec(nil, nil)); !(got == hardwareDefaultsV1beta2{}) { t.Fatalf("got %+v", got) } }) t.Run("complete hardware never asks the cluster", func(t *testing.T) { d := &fakeDiscoverer{found: DiscoveredHardware{SKU: "h100_sxm", GPUs: 4}} got := DGDRV1beta2Options{Hardware: d}.resolveHardware(context.Background(), admissionv1.Create, spec(v1beta2Ptr("b200_sxm"), v1beta2Ptr(int32(2)))) - if d.calls != 0 || !got.empty() { + if d.calls != 0 || !(got == hardwareDefaultsV1beta2{}) { t.Fatalf("calls=%d got=%+v", d.calls, got) } }) @@ -324,7 +332,7 @@ func TestResolveHardwareV1beta2(t *testing.T) { }) t.Run("a discovery failure leaves everything unset", func(t *testing.T) { d := &fakeDiscoverer{err: errors.New("no DCGM")} - if got := (DGDRV1beta2Options{Hardware: d}).resolveHardware(context.Background(), admissionv1.Create, spec(nil, nil)); !got.empty() { + if got := (DGDRV1beta2Options{Hardware: d}).resolveHardware(context.Background(), admissionv1.Create, spec(nil, nil)); !(got == hardwareDefaultsV1beta2{}) { t.Fatalf("got %+v", got) } }) @@ -422,7 +430,7 @@ func TestHardwareIsDiscoveredOnlyOnCreate(t *testing.T) { t.Run("operation "+string(op), func(t *testing.T) { d := &fakeDiscoverer{found: DiscoveredHardware{SKU: "h200_sxm", GPUs: 8}} options := DGDRV1beta2Options{Hardware: d} - if got := options.resolveHardware(context.Background(), op, v1beta2SchemaDefaulted()); !got.empty() { + if got := options.resolveHardware(context.Background(), op, v1beta2SchemaDefaulted()); !(got == hardwareDefaultsV1beta2{}) { t.Fatalf("hardware resolved on %q: %+v", op, got) } if d.calls != 0 { From 7d6283493bcb227af8ca11c0b56d2c8b58d30734 Mon Sep 17 00:00:00 2001 From: devivasudevan <49675305+devivasudevan@users.noreply.github.com> Date: Fri, 23 Oct 2026 10:47:57 +0200 Subject: [PATCH 31/40] feat(operator): re-run hardware discovery for v1beta2 updates that change spec.hardware Signed-off-by: devivasudevan <49675305+devivasudevan@users.noreply.github.com> --- .../dynamographdeploymentrequest_handler.go | 9 +- .../dynamographdeploymentrequest_v1beta2.go | 33 ++++- ...ographdeploymentrequest_v1beta2_helpers.go | 25 ++-- ...namographdeploymentrequest_v1beta2_test.go | 137 +++++++++++++++--- ...ploymentrequest_validation_envtest_test.go | 69 +++++++++ 5 files changed, 234 insertions(+), 39 deletions(-) diff --git a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_handler.go b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_handler.go index 8826650e166c..03e6474f940d 100644 --- a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_handler.go +++ b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_handler.go @@ -103,9 +103,14 @@ func (d *DGDRDefaulter) Default(ctx context.Context, obj runtime.Object) error { } // The original request version decides what spec.v1beta2 means. A native v1beta2 Request - // is defaulted from its carrier on CREATE and UPDATE; only CREATE reads the cluster. + // is defaulted from its carrier on CREATE and UPDATE; only CREATE + // and an UPDATE that changes spec.hardware read the cluster. if internalwebhook.RequestUsesV1Beta2(req) && dgdr.Spec.V1Beta2 != nil { - return defaultDynamoGraphDeploymentRequestV1beta2(ctx, dgdr, req.Operation, d.v1beta2) + oldSpec, err := decodeOldSpecV1beta2(req.Operation, req.OldObject.Raw) + if err != nil { + return err + } + return defaultDynamoGraphDeploymentRequestV1beta2(ctx, dgdr, req.Operation, oldSpec, d.v1beta2) } // A foreign-version view (a carrier written or updated through the wrong API version, or // a v1beta1 payload written through v1beta2) is read-only; validation enforces that. diff --git a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2.go b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2.go index cad0e0caf3f2..e292358a95db 100644 --- a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2.go +++ b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2.go @@ -57,15 +57,16 @@ const ( // values later, when the input is lowered. // defaultDynamoGraphDeploymentRequestV1beta2 defaults the v1beta2 spec that dgdr carries in -// spec.v1beta2. The environment is read once, before any mutation, and only on CREATE: -// discovered hardware is persisted with the Request, so a later write never replaces what -// the Request was created with. The envelope is rewritten +// spec.v1beta2. The environment is read once, before any mutation, and only for a CREATE or +// an UPDATE that changes spec.hardware (see resolveHardware). oldSpec is the stored spec on +// UPDATE and nil otherwise. The envelope is rewritten // only when defaulting changed the spec, so a defaulted object stays byte-identical across // repeated admissions. func defaultDynamoGraphDeploymentRequestV1beta2( ctx context.Context, dgdr *nvidiacomv1beta1.DynamoGraphDeploymentRequest, operation admissionv1.Operation, + oldSpec *nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec, options DGDRV1beta2Options, ) error { var spec nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec @@ -77,7 +78,7 @@ func defaultDynamoGraphDeploymentRequestV1beta2( return fmt.Errorf("encode spec.v1beta2: %w", err) } - defaultDynamoGraphDeploymentRequestSpecV1beta2(&spec, options.resolveDefaults(ctx, operation, &spec)) + defaultDynamoGraphDeploymentRequestSpecV1beta2(&spec, options.resolveDefaults(ctx, operation, oldSpec, &spec)) after, err := json.Marshal(&spec) if err != nil { @@ -89,6 +90,30 @@ func defaultDynamoGraphDeploymentRequestV1beta2( return nil } +// decodeOldSpecV1beta2 returns the stored v1beta2 spec an UPDATE replaces. The API server +// converts the old object to the same v1beta1 carrier view as the new one, so the spec sits +// in the same spec.v1beta2 envelope. It returns nil when there is nothing to compare with. +func decodeOldSpecV1beta2( + operation admissionv1.Operation, + oldObject []byte, +) (*nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec, error) { + if operation != admissionv1.Update || len(oldObject) == 0 { + return nil, nil + } + var old nvidiacomv1beta1.DynamoGraphDeploymentRequest + if err := json.Unmarshal(oldObject, &old); err != nil { + return nil, fmt.Errorf("decode old object: %w", err) + } + if old.Spec.V1Beta2 == nil { + return nil, nil + } + var spec nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec + if err := json.Unmarshal(old.Spec.V1Beta2.Raw, &spec); err != nil { + return nil, fmt.Errorf("decode old spec.v1beta2: %w", err) + } + return &spec, nil +} + // dynamoGraphDeploymentRequestV1beta2DefaultOptions carries the request-scoped values the // traversal needs. An empty field means the environment had nothing to offer. type dynamoGraphDeploymentRequestV1beta2DefaultOptions struct { diff --git a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_helpers.go b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_helpers.go index 56271a21b82b..79180b71c12b 100644 --- a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_helpers.go +++ b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_helpers.go @@ -25,6 +25,7 @@ import ( nvidiacomv1beta2 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta2" "github.com/ai-dynamo/dynamo/deploy/operator/internal/gpu" admissionv1 "k8s.io/api/admission/v1" + apiequality "k8s.io/apimachinery/pkg/api/equality" "sigs.k8s.io/controller-runtime/pkg/log" ) @@ -75,26 +76,32 @@ type DGDRV1beta2Options struct { func (o DGDRV1beta2Options) resolveDefaults( ctx context.Context, operation admissionv1.Operation, - spec *nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec, + oldSpec, spec *nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec, ) dynamoGraphDeploymentRequestV1beta2DefaultOptions { return dynamoGraphDeploymentRequestV1beta2DefaultOptions{ - hardware: o.resolveHardware(ctx, operation, spec), + hardware: o.resolveHardware(ctx, operation, oldSpec, spec), aisimulateImage: o.AISimulateImage, } } -// resolveHardware returns the SKU and budget the typed spec omits on CREATE. It asks the cluster only +// resolveHardware returns the SKU and budget the typed spec omits. It asks the cluster only // when something is missing, restricted to the typed SKU when there is one. +// +// Discovery runs on CREATE and on an UPDATE that changes spec.hardware. A Request stays +// mutable and every Run snapshots it, so a caller may change or remove typed hardware before +// the next Run. An UPDATE that leaves spec.hardware unchanged, metadata-only ones included, +// never asks the cluster: a Request created while discovery had nothing to offer must not +// pick hardware up once discovery recovers, which would change its spec and generation behind +// the writer's back. oldSpec is the stored spec on UPDATE and nil otherwise. The decision reads +// the typed hardware alone, never spec.aisimulate.parameters. func (o DGDRV1beta2Options) resolveHardware( ctx context.Context, operation admissionv1.Operation, - spec *nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec, + oldSpec, spec *nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec, ) hardwareDefaultsV1beta2 { - // Discovery is a CREATE-time decision. A Request that omitted its hardware and was - // created while discovery had nothing to offer must not pick hardware up on a later - // metadata-only update: that would change the spec, and its generation, behind the - // writer's back. Any other operation, known or not, leaves the hardware alone. - if operation != admissionv1.Create || o.Hardware == nil { + changed := operation == admissionv1.Update && oldSpec != nil && + !apiequality.Semantic.DeepEqual(oldSpec.Hardware, spec.Hardware) + if o.Hardware == nil || (operation != admissionv1.Create && !changed) { return hardwareDefaultsV1beta2{} } var sku string diff --git a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_test.go b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_test.go index 55b90fabff49..e7fb52a9f596 100644 --- a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_test.go +++ b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_test.go @@ -280,20 +280,20 @@ func TestResolveHardwareV1beta2(t *testing.T) { } t.Run("no discoverer", func(t *testing.T) { - if got := (DGDRV1beta2Options{}).resolveHardware(context.Background(), admissionv1.Create, spec(nil, nil)); !(got == hardwareDefaultsV1beta2{}) { + if got := (DGDRV1beta2Options{}).resolveHardware(context.Background(), admissionv1.Create, nil, spec(nil, nil)); !(got == hardwareDefaultsV1beta2{}) { t.Fatalf("got %+v", got) } }) t.Run("complete hardware never asks the cluster", func(t *testing.T) { d := &fakeDiscoverer{found: DiscoveredHardware{SKU: "h100_sxm", GPUs: 4}} - got := DGDRV1beta2Options{Hardware: d}.resolveHardware(context.Background(), admissionv1.Create, spec(v1beta2Ptr("b200_sxm"), v1beta2Ptr(int32(2)))) + got := DGDRV1beta2Options{Hardware: d}.resolveHardware(context.Background(), admissionv1.Create, nil, spec(v1beta2Ptr("b200_sxm"), v1beta2Ptr(int32(2)))) if d.calls != 0 || !(got == hardwareDefaultsV1beta2{}) { t.Fatalf("calls=%d got=%+v", d.calls, got) } }) t.Run("everything missing", func(t *testing.T) { d := &fakeDiscoverer{found: DiscoveredHardware{SKU: "h200_sxm", GPUs: 16}} - got := DGDRV1beta2Options{Hardware: d}.resolveHardware(context.Background(), admissionv1.Create, spec(nil, nil)) + got := DGDRV1beta2Options{Hardware: d}.resolveHardware(context.Background(), admissionv1.Create, nil, spec(nil, nil)) if got.sku != "h200_sxm" || got.budget != 16 || d.filter != "" { t.Fatalf("got %+v, filter %q", got, d.filter) } @@ -303,42 +303,42 @@ func TestResolveHardwareV1beta2(t *testing.T) { }) t.Run("a pinned SKU filters discovery and keeps its value", func(t *testing.T) { d := &fakeDiscoverer{found: DiscoveredHardware{SKU: "b200_sxm", GPUs: 4}} - got := DGDRV1beta2Options{Hardware: d}.resolveHardware(context.Background(), admissionv1.Create, spec(v1beta2Ptr("b200_sxm"), nil)) + got := DGDRV1beta2Options{Hardware: d}.resolveHardware(context.Background(), admissionv1.Create, nil, spec(v1beta2Ptr("b200_sxm"), nil)) if d.filter != "b200_sxm" || got.sku != "" || got.budget != 4 { t.Fatalf("got %+v, filter %q", got, d.filter) } }) t.Run("a pinned budget keeps its value", func(t *testing.T) { d := &fakeDiscoverer{found: DiscoveredHardware{SKU: "h200_sxm", GPUs: 64}} - got := DGDRV1beta2Options{Hardware: d}.resolveHardware(context.Background(), admissionv1.Create, spec(nil, v1beta2Ptr(int32(3)))) + got := DGDRV1beta2Options{Hardware: d}.resolveHardware(context.Background(), admissionv1.Create, nil, spec(nil, v1beta2Ptr(int32(3)))) if got.sku != "h200_sxm" || got.budget != 0 { t.Fatalf("got %+v", got) } }) t.Run("the budget is capped", func(t *testing.T) { d := &fakeDiscoverer{found: DiscoveredHardware{SKU: "h200_sxm", GPUs: 1024}} - if got := (DGDRV1beta2Options{Hardware: d}).resolveHardware(context.Background(), admissionv1.Create, spec(nil, nil)); got.budget != 32 { + if got := (DGDRV1beta2Options{Hardware: d}).resolveHardware(context.Background(), admissionv1.Create, nil, spec(nil, nil)); got.budget != 32 { t.Fatalf("default cap: %+v", got) } - if got := (DGDRV1beta2Options{Hardware: d, MaxHardwareBudget: 8}).resolveHardware(context.Background(), admissionv1.Create, spec(nil, nil)); got.budget != 8 { + if got := (DGDRV1beta2Options{Hardware: d, MaxHardwareBudget: 8}).resolveHardware(context.Background(), admissionv1.Create, nil, spec(nil, nil)); got.budget != 8 { t.Fatalf("explicit cap: %+v", got) } }) t.Run("an unknown GPU model yields no SKU", func(t *testing.T) { d := &fakeDiscoverer{found: DiscoveredHardware{GPUs: 8}} - if got := (DGDRV1beta2Options{Hardware: d}).resolveHardware(context.Background(), admissionv1.Create, spec(nil, nil)); got.sku != "" || got.budget != 8 { + if got := (DGDRV1beta2Options{Hardware: d}).resolveHardware(context.Background(), admissionv1.Create, nil, spec(nil, nil)); got.sku != "" || got.budget != 8 { t.Fatalf("got %+v", got) } }) t.Run("a discovery failure leaves everything unset", func(t *testing.T) { d := &fakeDiscoverer{err: errors.New("no DCGM")} - if got := (DGDRV1beta2Options{Hardware: d}).resolveHardware(context.Background(), admissionv1.Create, spec(nil, nil)); !(got == hardwareDefaultsV1beta2{}) { + if got := (DGDRV1beta2Options{Hardware: d}).resolveHardware(context.Background(), admissionv1.Create, nil, spec(nil, nil)); !(got == hardwareDefaultsV1beta2{}) { t.Fatalf("got %+v", got) } }) t.Run("the timeout is configurable", func(t *testing.T) { d := &fakeDiscoverer{found: DiscoveredHardware{GPUs: 1}} - DGDRV1beta2Options{Hardware: d, HardwareDiscoveryTimeout: time.Hour}.resolveHardware(context.Background(), admissionv1.Create, spec(nil, nil)) + DGDRV1beta2Options{Hardware: d, HardwareDiscoveryTimeout: time.Hour}.resolveHardware(context.Background(), admissionv1.Create, nil, spec(nil, nil)) if d.timeout < time.Minute { t.Fatalf("timeout = %v", d.timeout) } @@ -348,7 +348,7 @@ func TestResolveHardwareV1beta2(t *testing.T) { func TestResolveDefaultsV1beta2(t *testing.T) { d := &fakeDiscoverer{found: DiscoveredHardware{SKU: "h200_sxm", GPUs: 8}} got := DGDRV1beta2Options{Hardware: d, AISimulateImage: "registry.example.com/aisimulate:1.2.0"}. - resolveDefaults(context.Background(), admissionv1.Create, v1beta2SchemaDefaulted()) + resolveDefaults(context.Background(), admissionv1.Create, nil, v1beta2SchemaDefaulted()) if got.aisimulateImage != "registry.example.com/aisimulate:1.2.0" || got.hardware.sku != "h200_sxm" || got.hardware.budget != 8 { t.Fatalf("got %+v", got) } @@ -372,7 +372,7 @@ func TestDefaultEnvelopeV1beta2(t *testing.T) { } dgdr := v1beta2Envelope(t, v1beta2SchemaDefaulted()) - if err := defaultDynamoGraphDeploymentRequestV1beta2(context.Background(), dgdr, admissionv1.Create, options); err != nil { + if err := defaultDynamoGraphDeploymentRequestV1beta2(context.Background(), dgdr, admissionv1.Create, nil, options); err != nil { t.Fatal(err) } var got nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec @@ -391,7 +391,7 @@ func TestDefaultEnvelopeV1beta2(t *testing.T) { // A second admission of the defaulted object changes nothing, byte for byte. first := string(dgdr.Spec.V1Beta2.Raw) - if err := defaultDynamoGraphDeploymentRequestV1beta2(context.Background(), dgdr, admissionv1.Create, options); err != nil { + if err := defaultDynamoGraphDeploymentRequestV1beta2(context.Background(), dgdr, admissionv1.Create, nil, options); err != nil { t.Fatal(err) } if string(dgdr.Spec.V1Beta2.Raw) != first { @@ -409,7 +409,7 @@ func TestDefaultEnvelopeV1beta2LeavesUnchangedEnvelopeUntouched(t *testing.T) { dgdr := v1beta2Envelope(t, spec) dgdr.Spec.V1Beta2.Raw = append([]byte(" "), raw...) original := string(dgdr.Spec.V1Beta2.Raw) - if err := defaultDynamoGraphDeploymentRequestV1beta2(context.Background(), dgdr, admissionv1.Create, DGDRV1beta2Options{}); err != nil { + if err := defaultDynamoGraphDeploymentRequestV1beta2(context.Background(), dgdr, admissionv1.Create, nil, DGDRV1beta2Options{}); err != nil { t.Fatal(err) } if string(dgdr.Spec.V1Beta2.Raw) != original { @@ -420,21 +420,73 @@ func TestDefaultEnvelopeV1beta2LeavesUnchangedEnvelopeUntouched(t *testing.T) { func TestDefaultEnvelopeV1beta2RejectsMalformedSpec(t *testing.T) { dgdr := &nvidiacomv1beta1.DynamoGraphDeploymentRequest{} dgdr.Spec.V1Beta2 = &apiextensionsv1.JSON{Raw: []byte(`{"model": 3}`)} - if err := defaultDynamoGraphDeploymentRequestV1beta2(context.Background(), dgdr, admissionv1.Create, DGDRV1beta2Options{}); err == nil { + if err := defaultDynamoGraphDeploymentRequestV1beta2(context.Background(), dgdr, admissionv1.Create, nil, DGDRV1beta2Options{}); err == nil { t.Fatal("want a decode error") } } -func TestHardwareIsDiscoveredOnlyOnCreate(t *testing.T) { - for _, op := range []admissionv1.Operation{admissionv1.Update, admissionv1.Delete, admissionv1.Connect, ""} { +func hardwareSpecV1beta2(sku *string, budget *int32) *nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec { + spec := v1beta2SchemaDefaulted() + if sku != nil || budget != nil { + spec.Hardware = &nvidiacomv1beta2.HardwareSpec{GPU: &nvidiacomv1beta2.GPUHardwareSpec{SKU: sku, Budget: budget}} + } + return spec +} + +func TestHardwareIsNeverDiscoveredOnOtherOperations(t *testing.T) { + for _, op := range []admissionv1.Operation{admissionv1.Delete, admissionv1.Connect, ""} { t.Run("operation "+string(op), func(t *testing.T) { d := &fakeDiscoverer{found: DiscoveredHardware{SKU: "h200_sxm", GPUs: 8}} - options := DGDRV1beta2Options{Hardware: d} - if got := options.resolveHardware(context.Background(), op, v1beta2SchemaDefaulted()); !(got == hardwareDefaultsV1beta2{}) { - t.Fatalf("hardware resolved on %q: %+v", op, got) + old := hardwareSpecV1beta2(v1beta2Ptr("h100_sxm"), v1beta2Ptr(int32(4))) + got := DGDRV1beta2Options{Hardware: d}.resolveHardware(context.Background(), op, old, hardwareSpecV1beta2(nil, nil)) + if got != (hardwareDefaultsV1beta2{}) || d.calls != 0 { + t.Fatalf("hardware resolved on %q: %+v, calls %d", op, got, d.calls) } - if d.calls != 0 { - t.Fatalf("the cluster was asked on %q", op) + }) + } +} + +func TestResolveHardwareByOperationV1beta2(t *testing.T) { + full := func() *nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec { + return hardwareSpecV1beta2(v1beta2Ptr("h100_sxm"), v1beta2Ptr(int32(4))) + } + tests := []struct { + name string + op admissionv1.Operation + old, spec *nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec + want hardwareDefaultsV1beta2 + wantCalls int + wantFilter string + }{ + {"create discovers everything", admissionv1.Create, nil, hardwareSpecV1beta2(nil, nil), + hardwareDefaultsV1beta2{sku: "h200_sxm", budget: 8}, 1, ""}, + {"metadata-only update without hardware does not discover", admissionv1.Update, + hardwareSpecV1beta2(nil, nil), hardwareSpecV1beta2(nil, nil), hardwareDefaultsV1beta2{}, 0, ""}, + {"update with unchanged incomplete hardware does not discover", admissionv1.Update, + hardwareSpecV1beta2(v1beta2Ptr("h100_sxm"), nil), hardwareSpecV1beta2(v1beta2Ptr("h100_sxm"), nil), + hardwareDefaultsV1beta2{}, 0, ""}, + {"update without a stored spec does not discover", admissionv1.Update, + nil, hardwareSpecV1beta2(nil, nil), hardwareDefaultsV1beta2{}, 0, ""}, + {"removing only the budget discovers only the budget", admissionv1.Update, + full(), hardwareSpecV1beta2(v1beta2Ptr("h100_sxm"), nil), + hardwareDefaultsV1beta2{budget: 8}, 1, "h100_sxm"}, + {"removing only the SKU discovers only the SKU", admissionv1.Update, + full(), hardwareSpecV1beta2(nil, v1beta2Ptr(int32(4))), + hardwareDefaultsV1beta2{sku: "h200_sxm"}, 1, ""}, + {"removing all typed hardware discovers both", admissionv1.Update, + full(), hardwareSpecV1beta2(nil, nil), + hardwareDefaultsV1beta2{sku: "h200_sxm", budget: 8}, 1, ""}, + {"changing to complete hardware does not discover", admissionv1.Update, + full(), hardwareSpecV1beta2(v1beta2Ptr("b200_sxm"), v1beta2Ptr(int32(2))), + hardwareDefaultsV1beta2{}, 0, ""}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + d := &fakeDiscoverer{found: DiscoveredHardware{SKU: "h200_sxm", GPUs: 8}} + got := DGDRV1beta2Options{Hardware: d}.resolveHardware(context.Background(), tt.op, tt.old, tt.spec) + if got != tt.want || d.calls != tt.wantCalls || (tt.wantCalls > 0 && d.filter != tt.wantFilter) { + t.Fatalf("got %+v, calls %d, filter %q; want %+v, calls %d, filter %q", + got, d.calls, d.filter, tt.want, tt.wantCalls, tt.wantFilter) } }) } @@ -442,7 +494,7 @@ func TestHardwareIsDiscoveredOnlyOnCreate(t *testing.T) { // A Request created while discovery had nothing to offer keeps its spec through a later // metadata-only update, even after the cluster recovers. -func TestUpdateKeepsHardwareUnresolved(t *testing.T) { +func TestUpdateKeepsUnchangedHardwareUnresolved(t *testing.T) { options := DGDRV1beta2Options{Hardware: &fakeDiscoverer{found: DiscoveredHardware{SKU: "h200_sxm", GPUs: 8}}} // The objective metric is still defaulted on update, so the spec already carries it. spec := v1beta2SchemaDefaulted() @@ -450,7 +502,8 @@ func TestUpdateKeepsHardwareUnresolved(t *testing.T) { spec.Objective.Metric = &metric dgdr := v1beta2Envelope(t, spec) first := string(dgdr.Spec.V1Beta2.Raw) - if err := defaultDynamoGraphDeploymentRequestV1beta2(context.Background(), dgdr, admissionv1.Update, options); err != nil { + if err := defaultDynamoGraphDeploymentRequestV1beta2( + context.Background(), dgdr, admissionv1.Update, hardwareSpecV1beta2(nil, nil), options); err != nil { t.Fatal(err) } if string(dgdr.Spec.V1Beta2.Raw) != first { @@ -458,6 +511,42 @@ func TestUpdateKeepsHardwareUnresolved(t *testing.T) { } } +func TestUpdateRediscoversRemovedHardware(t *testing.T) { + options := DGDRV1beta2Options{Hardware: &fakeDiscoverer{found: DiscoveredHardware{SKU: "h200_sxm", GPUs: 8}}} + dgdr := v1beta2Envelope(t, v1beta2SchemaDefaulted()) + old := hardwareSpecV1beta2(v1beta2Ptr("h100_sxm"), v1beta2Ptr(int32(4))) + if err := defaultDynamoGraphDeploymentRequestV1beta2(context.Background(), dgdr, admissionv1.Update, old, options); err != nil { + t.Fatal(err) + } + var got nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec + if err := json.Unmarshal(dgdr.Spec.V1Beta2.Raw, &got); err != nil { + t.Fatal(err) + } + if got.Hardware == nil || got.Hardware.GPU == nil || *got.Hardware.GPU.SKU != "h200_sxm" || *got.Hardware.GPU.Budget != 8 { + t.Fatalf("hardware not rediscovered: %+v", got.Hardware) + } +} + +func TestDecodeOldSpecV1beta2(t *testing.T) { + old, err := json.Marshal(v1beta2Envelope(t, hardwareSpecV1beta2(v1beta2Ptr("h100_sxm"), nil))) + if err != nil { + t.Fatal(err) + } + got, err := decodeOldSpecV1beta2(admissionv1.Update, old) + if err != nil || got == nil || got.Hardware == nil || *got.Hardware.GPU.SKU != "h100_sxm" { + t.Fatalf("got %+v, err %v", got, err) + } + if got, err := decodeOldSpecV1beta2(admissionv1.Create, old); got != nil || err != nil { + t.Fatalf("create has no old spec: %+v, %v", got, err) + } + if got, err := decodeOldSpecV1beta2(admissionv1.Update, []byte(`{}`)); got != nil || err != nil { + t.Fatalf("an old object without the envelope has no old spec: %+v, %v", got, err) + } + if _, err := decodeOldSpecV1beta2(admissionv1.Update, []byte(`{`)); err == nil { + t.Fatal("want a decode error") + } +} + func TestDefaultSpecV1beta2StructuralDefaults(t *testing.T) { opts := dynamoGraphDeploymentRequestV1beta2DefaultOptions{} diff --git a/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_validation_envtest_test.go b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_validation_envtest_test.go index c8583be88741..232f4f3db024 100644 --- a/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_validation_envtest_test.go +++ b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_validation_envtest_test.go @@ -610,6 +610,64 @@ func TestDynamoGraphDeploymentRequestValidator_Validate(t *testing.T) { } }, }, + { + name: "v1beta2 metadata-only update does not rediscover hardware after discovery recovers", + oldRequest: v1beta2DGDRForAdmission(func(spec map[string]any) { + setV1beta2SearchSpace(spec, map[string]any{"hardware_sku": "h100_sxm", "gpu_budget": int64(8)}) + }, nil), + request: v1beta2DGDRForAdmission(func(spec map[string]any) { + setV1beta2SearchSpace(spec, map[string]any{"hardware_sku": "h100_sxm", "gpu_budget": int64(8)}) + }, map[string]any{"updated": "true"}), + gpuDiscovery: true, + discoveryErr: errors.New("no GPUs found"), + beforeUpdate: func(*testing.T) { admissionHardware.set(discoveredH200, nil) }, + check: func(t *testing.T, result *unstructured.Unstructured) { + v1beta2Absent(t, result, "spec", "hardware") + if got := result.GetGeneration(); got != 1 { + t.Fatalf("generation = %d, want 1: the spec must not change", got) + } + }, + }, + { + name: "v1beta2 removing only the budget rediscovers only the budget", + oldRequest: v1beta2DGDRForAdmission(func(spec map[string]any) { + spec["hardware"] = map[string]any{"gpu": map[string]any{"sku": "h100_sxm", "budget": int64(4)}} + }, nil), + request: v1beta2DGDRForAdmission(func(spec map[string]any) { + spec["hardware"] = map[string]any{"gpu": map[string]any{"sku": "h100_sxm"}} + }, nil), + gpuDiscovery: true, + discovered: discoveredH200, + check: func(t *testing.T, result *unstructured.Unstructured) { + expectV1beta2GPU(t, result, "h100_sxm", 8) + }, + }, + { + name: "v1beta2 removing only the SKU rediscovers only the SKU", + oldRequest: v1beta2DGDRForAdmission(func(spec map[string]any) { + spec["hardware"] = map[string]any{"gpu": map[string]any{"sku": "h100_sxm", "budget": int64(4)}} + }, nil), + request: v1beta2DGDRForAdmission(func(spec map[string]any) { + spec["hardware"] = map[string]any{"gpu": map[string]any{"budget": int64(4)}} + }, nil), + gpuDiscovery: true, + discovered: discoveredH200, + check: func(t *testing.T, result *unstructured.Unstructured) { + expectV1beta2GPU(t, result, "h200_sxm", 4) + }, + }, + { + name: "v1beta2 removing all typed hardware rediscovers the SKU and the budget", + oldRequest: v1beta2DGDRForAdmission(func(spec map[string]any) { + spec["hardware"] = map[string]any{"gpu": map[string]any{"sku": "h100_sxm", "budget": int64(4)}} + }, nil), + request: v1beta2DGDRForAdmission(nil, nil), + gpuDiscovery: true, + discovered: discoveredH200, + check: func(t *testing.T, result *unstructured.Unstructured) { + expectV1beta2GPU(t, result, "h200_sxm", 8) + }, + }, { name: "v1beta2 empty workload receives the static model-only default", request: v1beta2DGDRForAdmission(func(spec map[string]any) { @@ -1187,3 +1245,14 @@ func setBeta2AdmissionHardware(request *nvidiacomv1beta2.DynamoGraphDeploymentRe GPU: &nvidiacomv1beta2.GPUHardwareSpec{SKU: &sku, Budget: &budget}, } } + +// expectV1beta2GPU asserts the typed GPU hardware the API server stored. +func expectV1beta2GPU(t *testing.T, result *unstructured.Unstructured, sku string, budget float64) { + t.Helper() + if got := v1beta2String(t, result, "spec", "hardware", "gpu", "sku"); got != sku { + t.Fatalf("hardware.gpu.sku = %q, want %q", got, sku) + } + if got := v1beta2Number(t, result, "spec", "hardware", "gpu", "budget"); got != budget { + t.Fatalf("hardware.gpu.budget = %v, want %v", got, budget) + } +} From 59d13188ad501fe85fe3f9261ddd76b4cfa31046 Mon Sep 17 00:00:00 2001 From: devivasudevan <49675305+devivasudevan@users.noreply.github.com> Date: Fri, 23 Oct 2026 11:04:40 +0200 Subject: [PATCH 32/40] fix(operator): validate native v1beta2 specs on every update Signed-off-by: devivasudevan <49675305+devivasudevan@users.noreply.github.com> --- .../validation/dynamographdeploymentrequest.go | 9 +++------ ...raphdeploymentrequest_validation_envtest_test.go | 13 +++++++++++++ 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest.go b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest.go index dc31bcc88445..3315b26d7bf7 100644 --- a/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest.go +++ b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest.go @@ -260,13 +260,10 @@ func (v *dynamoGraphDeploymentRequestValidation) validateDynamoGraphDeploymentRe } return foreignUpdateErrors(fldPath, newSpec.V1Beta2 != nil) } - // A native v1beta2 spec is unconstrained by phase: an unchanged spec is always allowed, so - // status-preserving writes and defaulting-only updates pass, and any change is fully - // validated as if it were new. + // A native v1beta2 spec is unconstrained by phase. Stateless validation of the new typed + // spec runs on every CREATE and UPDATE: an unchanged spec is no reason to skip it. Ratcheting + // belongs to the generated AISimulate schema validation, which will ratchet its own result. if newSpec.V1Beta2 != nil { - if apiequality.Semantic.DeepEqual(newSpec, oldSpec) { - return nil - } return validateDynamoGraphDeploymentRequestEnvelopeV1beta2(newSpec.V1Beta2, fldPath.Child("v1beta2")) } diff --git a/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_validation_envtest_test.go b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_validation_envtest_test.go index 232f4f3db024..3c8bd04d4951 100644 --- a/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_validation_envtest_test.go +++ b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_validation_envtest_test.go @@ -739,6 +739,19 @@ func TestDynamoGraphDeploymentRequestValidator_Validate(t *testing.T) { } }, }, + { + name: "v1beta2 stored invalid spec is rejected again on a metadata-only update", + oldRequest: v1beta2DGDRForAdmission(func(spec map[string]any) { + spec["backends"] = []any{"vllm", "vllm"} + }, nil), + request: v1beta2DGDRForAdmission(func(spec map[string]any) { + spec["backends"] = []any{"vllm", "vllm"} + }, map[string]any{"updated": "true"}), + seedWithoutWebhook: true, + gpuDiscovery: true, + discovered: discoveredH200, + wantWebhook: []string{`spec.backends[1]: Duplicate value: "vllm"`}, + }, { name: "v1beta2 duplicate backends are rejected on create", request: v1beta2DGDRForAdmission(func(spec map[string]any) { From dc5eee6cb1b3299f3230aca4c6084b03ce23cfc9 Mon Sep 17 00:00:00 2001 From: devivasudevan <49675305+devivasudevan@users.noreply.github.com> Date: Fri, 23 Oct 2026 11:21:40 +0200 Subject: [PATCH 33/40] refactor(operator): mirror the API tree in v1beta2 defaulting and share the GPU discoverer Default through the NativeSpec and NativeSearchInput types in declaration order. Make the aisimulate native paths package-private and drop the unused ones. HardwareDiscoverer now has the signature of gpu.Discoverer, so the shared discoverer satisfies it directly and the catalog mapping lives in resolveHardware. Signed-off-by: devivasudevan <49675305+devivasudevan@users.noreply.github.com> --- .../dynamographdeploymentrequest_types.go | 2 +- deploy/operator/cmd/main.go | 4 +- ...dia.com_dynamographdeploymentrequests.yaml | 2 +- .../nvidia.com_dynamographdeploymentruns.yaml | 2 +- .../controller/aisimulate/defaults.go | 18 +++--- .../internal/controller/aisimulate/input.go | 34 +++++------ .../internal/controller/aisimulate/paths.go | 54 ++++++++--------- .../internal/controller/suite_envtest_test.go | 6 +- .../dynamographdeploymentrequest_v1beta2.go | 59 ++++++++++++------- ...ographdeploymentrequest_v1beta2_helpers.go | 46 ++++----------- ...namographdeploymentrequest_v1beta2_test.go | 36 +++++------ ...ploymentrequest_validation_envtest_test.go | 8 +-- .../webhook/validation/suite_envtest_test.go | 10 ++-- 13 files changed, 137 insertions(+), 144 deletions(-) diff --git a/deploy/operator/api/v1beta2/dynamographdeploymentrequest_types.go b/deploy/operator/api/v1beta2/dynamographdeploymentrequest_types.go index 21bd766b4030..28a28e4bee2d 100644 --- a/deploy/operator/api/v1beta2/dynamographdeploymentrequest_types.go +++ b/deploy/operator/api/v1beta2/dynamographdeploymentrequest_types.go @@ -234,7 +234,7 @@ type NativeSearchInput struct { // Workload defines the synthetic traffic model used for every candidate evaluation. // When omitted, admission supplies a static shape of ISL 1024, OSL 128 and concurrency 10; - // an explicit empty workload receives the same shape from the schema. An explicit + // an explicit empty workload receives the same shape from admission. An explicit // workload.static: {} opts out of that shape, leaving the workload to // spec.aisimulate.parameters.workload. spec.aisimulate.parameters.workload overrides the // fields it names. diff --git a/deploy/operator/cmd/main.go b/deploy/operator/cmd/main.go index 3c762841c7a2..587a93c0c328 100644 --- a/deploy/operator/cmd/main.go +++ b/deploy/operator/cmd/main.go @@ -674,12 +674,12 @@ func registerWebhookHandlers( AISimulateImage: dgdrAISimulateDefaultImage, // The same discovery order as the DGDR reconciler: DCGM first, then node labels when // the GPUDiscovery gate is on. The webhook keeps its own short-lived cache. - Hardware: webhookdefaulting.NewGPUHardwareDiscoverer(&gpu.Discoverer{ + Hardware: &gpu.Discoverer{ Reader: mgr.GetAPIReader(), DCGM: gpu.NewGPUDiscovery(gpu.ScrapeMetricsEndpoint), Cache: gpu.NewGPUDiscoveryCache(), NodeLabels: func() bool { return gate.Enabled(features.GPUDiscovery) }, - }), + }, }, OperatorPrincipal: operatorPrincipal, Gate: gate, diff --git a/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentrequests.yaml b/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentrequests.yaml index 9a8e2621d652..2d143dfd67e5 100644 --- a/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentrequests.yaml +++ b/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentrequests.yaml @@ -9757,7 +9757,7 @@ spec: description: |- Workload defines the synthetic traffic model used for every candidate evaluation. When omitted, admission supplies a static shape of ISL 1024, OSL 128 and concurrency 10; - an explicit empty workload receives the same shape from the schema. An explicit + an explicit empty workload receives the same shape from admission. An explicit workload.static: {} opts out of that shape, leaving the workload to spec.aisimulate.parameters.workload. spec.aisimulate.parameters.workload overrides the fields it names. diff --git a/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentruns.yaml b/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentruns.yaml index ad6f33e55d3d..9a38dec2c50c 100644 --- a/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentruns.yaml +++ b/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentruns.yaml @@ -221,7 +221,7 @@ spec: description: |- Workload defines the synthetic traffic model used for every candidate evaluation. When omitted, admission supplies a static shape of ISL 1024, OSL 128 and concurrency 10; - an explicit empty workload receives the same shape from the schema. An explicit + an explicit empty workload receives the same shape from admission. An explicit workload.static: {} opts out of that shape, leaving the workload to spec.aisimulate.parameters.workload. spec.aisimulate.parameters.workload overrides the fields it names. diff --git a/deploy/operator/internal/controller/aisimulate/defaults.go b/deploy/operator/internal/controller/aisimulate/defaults.go index 5631a0eded41..8aecb2d8d569 100644 --- a/deploy/operator/internal/controller/aisimulate/defaults.go +++ b/deploy/operator/internal/controller/aisimulate/defaults.go @@ -39,25 +39,25 @@ const ( func applyDefaults(underlay map[string]any) error { b := builder{root: underlay} - if has(underlay, Workload...) { - if !has(underlay, WorkloadRequestCount...) && !has(underlay, WorkloadNumRequestRatio...) { - b.set(float64(DefaultRequestCount), WorkloadRequestCount) + if has(underlay, workloadPath...) { + if !has(underlay, workloadRequestCount...) && !has(underlay, workloadNumRequestRatio...) { + b.set(float64(DefaultRequestCount), workloadRequestCount) } - kvLoadRatio, _, _ := unstructured.NestedFieldNoCopy(underlay, WorkloadKVLoadRatio...) - if _, isRange := kvLoadRatio.([]any); isRange && !has(underlay, SweepMaxTrials...) { - b.set(float64(DefaultMaxTrials), SweepMaxTrials) + kvLoadRatio, _, _ := unstructured.NestedFieldNoCopy(underlay, workloadKVLoadRatio...) + if _, isRange := kvLoadRatio.([]any); isRange && !has(underlay, sweepMaxTrials...) { + b.set(float64(DefaultMaxTrials), sweepMaxTrials) } } - if hasSLABound(underlay) && !has(underlay, GoalStrictSLA...) { - b.set(true, GoalStrictSLA) + if hasSLABound(underlay) && !has(underlay, goalStrictSLA...) { + b.set(true, goalStrictSLA) } return b.err } // hasSLABound reports whether the input bounds at least one latency. func hasSLABound(input map[string]any) bool { - for _, path := range [][]string{GoalSLATTFTMs, GoalSLAITLMs, GoalSLAE2EMs} { + for _, path := range [][]string{goalSLATTFTMs, goalSLAITLMs, goalSLAE2EMs} { if bound, _, _ := unstructured.NestedFieldNoCopy(input, path...); bound != nil { return true } diff --git a/deploy/operator/internal/controller/aisimulate/input.go b/deploy/operator/internal/controller/aisimulate/input.go index 0b8bfc179bb6..fbfee0514080 100644 --- a/deploy/operator/internal/controller/aisimulate/input.go +++ b/deploy/operator/internal/controller/aisimulate/input.go @@ -88,72 +88,72 @@ func Underlay(spec *v1beta2.DynamoGraphDeploymentRequestSpec) (map[string]any, e b := builder{root: map[string]any{}} if spec.Model != nil { - b.set(spec.Model.Name, SearchSpaceModelName) + b.set(spec.Model.Name, searchSpaceModelName) } if len(spec.Backends) > 0 { backends := make([]string, len(spec.Backends)) for i, backend := range spec.Backends { backends[i] = string(backend) } - b.setStrings(backends, SearchSpaceBackend) + b.setStrings(backends, searchSpaceBackend) } if spec.Hardware != nil && spec.Hardware.GPU != nil { if sku := spec.Hardware.GPU.SKU; sku != nil && *sku != "" { - b.set(*sku, SearchSpaceHardwareSKU) + b.set(*sku, searchSpaceHardwareSKU) } if budget := spec.Hardware.GPU.Budget; budget != nil { - b.set(int64(*budget), SearchSpaceGPUBudget) + b.set(int64(*budget), searchSpaceGPUBudget) } } if spec.Workload != nil && spec.Workload.Static != nil { static := spec.Workload.Static if static.ISL != nil { - b.set(int64(*static.ISL), WorkloadISL) + b.set(int64(*static.ISL), workloadISL) } if static.OSL != nil { - b.set(int64(*static.OSL), WorkloadOSL) + b.set(int64(*static.OSL), workloadOSL) } if static.NumRequestRatio != nil { - b.set(*static.NumRequestRatio, WorkloadNumRequestRatio) + b.set(*static.NumRequestRatio, workloadNumRequestRatio) } if static.RequestRate != nil { - b.set(*static.RequestRate, WorkloadRequestRate) + b.set(*static.RequestRate, workloadRequestRate) } if static.Concurrency != nil { - b.set(int64(*static.Concurrency), WorkloadConcurrency) + b.set(int64(*static.Concurrency), workloadConcurrency) } switch len(static.KVLoadRatio) { case 0: case 1: - b.set(static.KVLoadRatio[0], WorkloadKVLoadRatio) + b.set(static.KVLoadRatio[0], workloadKVLoadRatio) default: ratios := make([]any, len(static.KVLoadRatio)) for i, ratio := range static.KVLoadRatio { ratios[i] = ratio } - b.set(ratios, WorkloadKVLoadRatio) + b.set(ratios, workloadKVLoadRatio) } } if objective := spec.Objective; objective != nil { if objective.Mode == v1beta2.ObjectiveModePareto { - b.set(TargetPareto, GoalTarget) + b.set(TargetPareto, goalTarget) if len(objective.Metrics) > 0 { - b.setStrings(nativeMetrics(objective.Metrics), GoalParetoObjectives) + b.setStrings(nativeMetrics(objective.Metrics), goalParetoObjectives) } } else if objective.Metric != nil { - b.set(nativeMetric(*objective.Metric), GoalTarget) + b.set(nativeMetric(*objective.Metric), goalTarget) } if sla := objective.SLA; sla != nil { if sla.TTFTMs != nil { - b.set(*sla.TTFTMs, GoalSLATTFTMs) + b.set(*sla.TTFTMs, goalSLATTFTMs) } if sla.ITLMs != nil { - b.set(*sla.ITLMs, GoalSLAITLMs) + b.set(*sla.ITLMs, goalSLAITLMs) } if sla.E2EMs != nil { - b.set(*sla.E2EMs, GoalSLAE2EMs) + b.set(*sla.E2EMs, goalSLAE2EMs) } } } diff --git a/deploy/operator/internal/controller/aisimulate/paths.go b/deploy/operator/internal/controller/aisimulate/paths.go index 6992859b49e0..bd2a9d16b7d8 100644 --- a/deploy/operator/internal/controller/aisimulate/paths.go +++ b/deploy/operator/internal/controller/aisimulate/paths.go @@ -19,36 +19,32 @@ package aisimulate import "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" -// Native input paths, in the snake_case spelling of the upstream AISimulate contract. -// They are read-only: callers pass them to the unstructured helpers and must not modify them. +// Native input paths, in the snake_case spelling of the upstream AISimulate contract. They +// address the same native key when the underlay is built and when derived defaults are read, +// and are passed to the unstructured helpers, which must not modify them. var ( - SearchSpaceModelName = []string{"search_space", "model_name"} - SearchSpaceBackend = []string{"search_space", "backend"} - SearchSpaceHardwareSKU = []string{"search_space", "hardware_sku"} - SearchSpaceGPUBudget = []string{"search_space", "gpu_budget"} - - Workload = []string{"workload"} - WorkloadISL = []string{"workload", "isl"} - WorkloadOSL = []string{"workload", "osl"} - WorkloadNumRequestRatio = []string{"workload", "num_request_ratio"} - WorkloadRequestRate = []string{"workload", "request_rate"} - WorkloadConcurrency = []string{"workload", "concurrency"} - WorkloadKVLoadRatio = []string{"workload", "kv_load_ratio"} - WorkloadRequestCount = []string{"workload", "request_count"} - WorkloadTracePath = []string{"workload", "trace_path"} - WorkloadTracePaths = []string{"workload", "trace_paths"} - - Goal = []string{"goal"} - GoalTarget = []string{"goal", "target"} - GoalParetoObjectives = []string{"goal", "pareto_objectives"} - GoalSLA = []string{"goal", "sla"} - GoalSLATTFTMs = []string{"goal", "sla", "ttft_ms"} - GoalSLAITLMs = []string{"goal", "sla", "itl_ms"} - GoalSLAE2EMs = []string{"goal", "sla", "e2e_ms"} - GoalStrictSLA = []string{"goal", "strict_sla"} - GoalMinGoodputRPS = []string{"goal", "min_goodput_rps"} - - SweepMaxTrials = []string{"sweep", "max_trials"} + searchSpaceModelName = []string{"search_space", "model_name"} + searchSpaceBackend = []string{"search_space", "backend"} + searchSpaceHardwareSKU = []string{"search_space", "hardware_sku"} + searchSpaceGPUBudget = []string{"search_space", "gpu_budget"} + + workloadPath = []string{"workload"} + workloadISL = []string{"workload", "isl"} + workloadOSL = []string{"workload", "osl"} + workloadNumRequestRatio = []string{"workload", "num_request_ratio"} + workloadRequestRate = []string{"workload", "request_rate"} + workloadConcurrency = []string{"workload", "concurrency"} + workloadKVLoadRatio = []string{"workload", "kv_load_ratio"} + workloadRequestCount = []string{"workload", "request_count"} + + goalTarget = []string{"goal", "target"} + goalParetoObjectives = []string{"goal", "pareto_objectives"} + goalSLATTFTMs = []string{"goal", "sla", "ttft_ms"} + goalSLAITLMs = []string{"goal", "sla", "itl_ms"} + goalSLAE2EMs = []string{"goal", "sla", "e2e_ms"} + goalStrictSLA = []string{"goal", "strict_sla"} + + sweepMaxTrials = []string{"sweep", "max_trials"} ) // TargetPareto is the goal.target of a multi-objective search. diff --git a/deploy/operator/internal/controller/suite_envtest_test.go b/deploy/operator/internal/controller/suite_envtest_test.go index 88ca0c68076b..0a6c4633cc74 100644 --- a/deploy/operator/internal/controller/suite_envtest_test.go +++ b/deploy/operator/internal/controller/suite_envtest_test.go @@ -13,6 +13,8 @@ import ( "os" "testing" + nvidiacomv1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/gpu" "github.com/ai-dynamo/dynamo/deploy/operator/internal/testing/operatorenv" webhookdefaulting "github.com/ai-dynamo/dynamo/deploy/operator/internal/webhook/defaulting" webhooksetup "github.com/ai-dynamo/dynamo/deploy/operator/internal/webhook/setup" @@ -32,8 +34,8 @@ const controllerAISimulateImage = "registry.example/aisimulate:1.2.0" type fixedHardwareDiscoverer struct{} -func (fixedHardwareDiscoverer) Discover(context.Context, string) (webhookdefaulting.DiscoveredHardware, error) { - return webhookdefaulting.DiscoveredHardware{SKU: "h200_sxm", GPUs: 8}, nil +func (fixedHardwareDiscoverer) Discover(context.Context, nvidiacomv1beta1.GPUSKUType) (*gpu.GPUInfo, error) { + return &gpu.GPUInfo{System: "h200_sxm", GPUsPerNode: 8, NodesWithGPUs: 1}, nil } var ( diff --git a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2.go b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2.go index e292358a95db..ce134ec09540 100644 --- a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2.go +++ b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2.go @@ -130,48 +130,63 @@ type hardwareDefaultsV1beta2 struct { } // defaultDynamoGraphDeploymentRequestSpecV1beta2 defaults spec in place. It applies on -// CREATE and UPDATE and is idempotent; opts carries no hardware on UPDATE. spec must not be -// nil. Explicit values, including empty objects, are kept. Children are visited in the -// declaration order of the API type. +// CREATE and UPDATE and is idempotent; opts carries hardware only when the environment was +// asked (see resolveHardware). spec must not be nil. Explicit values, including empty +// objects, are kept. The functions below mirror the Go API tree and visit children in +// declaration order. func defaultDynamoGraphDeploymentRequestSpecV1beta2( spec *nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec, opts dynamoGraphDeploymentRequestV1beta2DefaultOptions, ) { - if spec.Backends == nil { - spec.Backends = []nvidiacomv1beta2.Backend{"vllm", "sglang", "trtllm"} + defaultDynamoGraphDeploymentRequestNativeSpecV1beta2(&spec.DynamoGraphDeploymentRequestNativeSpec, opts) +} + +func defaultDynamoGraphDeploymentRequestNativeSpecV1beta2( + native *nvidiacomv1beta2.DynamoGraphDeploymentRequestNativeSpec, + opts dynamoGraphDeploymentRequestV1beta2DefaultOptions, +) { + defaultNativeSearchInputV1beta2(&native.NativeSearchInput, opts) +} + +func defaultNativeSearchInputV1beta2( + input *nvidiacomv1beta2.NativeSearchInput, + opts dynamoGraphDeploymentRequestV1beta2DefaultOptions, +) { + if input.Backends == nil { + input.Backends = []nvidiacomv1beta2.Backend{"vllm", "sglang", "trtllm"} } // Hardware has no static default: it exists only when the cluster supplied some. - if spec.Hardware == nil && opts.hardware != (hardwareDefaultsV1beta2{}) { - spec.Hardware = &nvidiacomv1beta2.HardwareSpec{} + if input.Hardware == nil && opts.hardware != (hardwareDefaultsV1beta2{}) { + input.Hardware = &nvidiacomv1beta2.HardwareSpec{} } - if spec.Hardware != nil { - defaultHardwareSpecV1beta2(spec.Hardware, opts.hardware) + if input.Hardware != nil { + defaultHardwareSpecV1beta2(input.Hardware, opts.hardware) } - if spec.Workload == nil { - spec.Workload = &nvidiacomv1beta2.WorkloadSpec{} + if input.Workload == nil { + input.Workload = &nvidiacomv1beta2.WorkloadSpec{} } - defaultWorkloadSpecV1beta2(spec.Workload) + defaultWorkloadSpecV1beta2(input.Workload) - if spec.Objective == nil { - spec.Objective = &nvidiacomv1beta2.ObjectiveSpec{} + if input.Objective == nil { + input.Objective = &nvidiacomv1beta2.ObjectiveSpec{} } - defaultObjectiveSpecV1beta2(spec.Objective) + defaultObjectiveSpecV1beta2(input.Objective) // The operator image is persisted with the Request, so spec.aisimulate is created to // carry it. - if spec.AISimulate == nil && opts.aisimulateImage != "" { - spec.AISimulate = &nvidiacomv1beta2.AISimulateSpec{} + if input.AISimulate == nil && opts.aisimulateImage != "" { + input.AISimulate = &nvidiacomv1beta2.AISimulateSpec{} } - if spec.AISimulate != nil { - defaultAISimulateSpecV1beta2(spec.AISimulate, opts.aisimulateImage) + if input.AISimulate != nil { + defaultAISimulateSpecV1beta2(input.AISimulate, opts.aisimulateImage) } - if spec.Recommendation == nil { - spec.Recommendation = &nvidiacomv1beta2.RecommendationSpec{} + if input.Recommendation == nil { + input.Recommendation = &nvidiacomv1beta2.RecommendationSpec{} } - defaultRecommendationSpecV1beta2(spec.Recommendation) + defaultRecommendationSpecV1beta2(input.Recommendation) } func defaultHardwareSpecV1beta2(hardware *nvidiacomv1beta2.HardwareSpec, defaults hardwareDefaultsV1beta2) { diff --git a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_helpers.go b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_helpers.go index 79180b71c12b..ed4b3adb0848 100644 --- a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_helpers.go +++ b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_helpers.go @@ -41,19 +41,10 @@ const ( defaultHardwareDiscoveryTimeout = 5 * time.Second ) -// DiscoveredHardware is what the cluster has, as far as it can be determined. -type DiscoveredHardware struct { - // SKU is the AISimulate hardware catalog name, or empty when the GPU model is not one - // the catalog knows. - SKU string - // GPUs is the total number of GPUs of that SKU in the cluster. - GPUs int -} - // HardwareDiscoverer finds the cluster's GPUs. filterSKU, when set, restricts the result to // GPUs of that SKU. type HardwareDiscoverer interface { - Discover(ctx context.Context, filterSKU string) (DiscoveredHardware, error) + Discover(ctx context.Context, filterSKU nvidiacomv1beta1.GPUSKUType) (*gpu.GPUInfo, error) } // DGDRV1beta2Options configures the defaults of v1beta2 Requests. @@ -123,43 +114,28 @@ func (o DGDRV1beta2Options) resolveHardware( ctx, cancel := context.WithTimeout(ctx, timeout) defer cancel() - found, err := o.Hardware.Discover(ctx, sku) + found, err := o.Hardware.Discover(ctx, nvidiacomv1beta1.GPUSKUType(sku)) if err != nil { log.FromContext(ctx).Info("hardware discovery failed; leaving hardware unresolved", "error", err.Error()) return hardwareDefaultsV1beta2{} } + // The cluster facts are mapped into the v1beta2 API here: the AISimulate catalog name of + // the GPU model (empty when the catalog does not know it) and the total GPU count. + var gpus int + if found != nil { + gpus = found.GPUsPerNode * found.NodesWithGPUs + } var defaults hardwareDefaultsV1beta2 if sku == "" { - defaults.sku = found.SKU + defaults.sku = string(gpu.KnownSKU(found)) } - if !hasBudget && found.GPUs > 0 { + if !hasBudget && gpus > 0 { maxBudget := o.MaxHardwareBudget if maxBudget <= 0 { maxBudget = defaultMaxHardwareBudget } - defaults.budget = int32(min(found.GPUs, int(maxBudget))) + defaults.budget = int32(min(gpus, int(maxBudget))) } return defaults } - -// gpuHardwareDiscoverer backs HardwareDiscoverer with the operator's GPU discovery. -type gpuHardwareDiscoverer struct { - discoverer *gpu.Discoverer -} - -// NewGPUHardwareDiscoverer adapts the operator's GPU discovery for hardware defaulting. -func NewGPUHardwareDiscoverer(discoverer *gpu.Discoverer) HardwareDiscoverer { - return gpuHardwareDiscoverer{discoverer: discoverer} -} - -func (g gpuHardwareDiscoverer) Discover(ctx context.Context, filterSKU string) (DiscoveredHardware, error) { - info, err := g.discoverer.Discover(ctx, nvidiacomv1beta1.GPUSKUType(filterSKU)) - if err != nil { - return DiscoveredHardware{}, err - } - return DiscoveredHardware{ - SKU: string(gpu.KnownSKU(info)), - GPUs: info.GPUsPerNode * info.NodesWithGPUs, - }, nil -} diff --git a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_test.go b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_test.go index e7fb52a9f596..e02001a815cc 100644 --- a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_test.go +++ b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_test.go @@ -27,6 +27,7 @@ import ( nvidiacomv1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" nvidiacomv1beta2 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta2" "github.com/ai-dynamo/dynamo/deploy/operator/internal/controller/aisimulate" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/gpu" admissionv1 "k8s.io/api/admission/v1" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" "k8s.io/apimachinery/pkg/runtime" @@ -254,20 +255,21 @@ func TestNativeTargetOverridesTheDefaultedMetric(t *testing.T) { } type fakeDiscoverer struct { - found DiscoveredHardware + found gpu.GPUInfo err error calls int filter string timeout time.Duration } -func (f *fakeDiscoverer) Discover(ctx context.Context, filterSKU string) (DiscoveredHardware, error) { +func (f *fakeDiscoverer) Discover(ctx context.Context, filterSKU nvidiacomv1beta1.GPUSKUType) (*gpu.GPUInfo, error) { f.calls++ - f.filter = filterSKU + f.filter = string(filterSKU) if deadline, ok := ctx.Deadline(); ok { f.timeout = time.Until(deadline) } - return f.found, f.err + found := f.found + return &found, f.err } func TestResolveHardwareV1beta2(t *testing.T) { @@ -285,14 +287,14 @@ func TestResolveHardwareV1beta2(t *testing.T) { } }) t.Run("complete hardware never asks the cluster", func(t *testing.T) { - d := &fakeDiscoverer{found: DiscoveredHardware{SKU: "h100_sxm", GPUs: 4}} + d := &fakeDiscoverer{found: gpu.GPUInfo{System: "h100_sxm", GPUsPerNode: 4, NodesWithGPUs: 1}} got := DGDRV1beta2Options{Hardware: d}.resolveHardware(context.Background(), admissionv1.Create, nil, spec(v1beta2Ptr("b200_sxm"), v1beta2Ptr(int32(2)))) if d.calls != 0 || !(got == hardwareDefaultsV1beta2{}) { t.Fatalf("calls=%d got=%+v", d.calls, got) } }) t.Run("everything missing", func(t *testing.T) { - d := &fakeDiscoverer{found: DiscoveredHardware{SKU: "h200_sxm", GPUs: 16}} + d := &fakeDiscoverer{found: gpu.GPUInfo{System: "h200_sxm", GPUsPerNode: 16, NodesWithGPUs: 1}} got := DGDRV1beta2Options{Hardware: d}.resolveHardware(context.Background(), admissionv1.Create, nil, spec(nil, nil)) if got.sku != "h200_sxm" || got.budget != 16 || d.filter != "" { t.Fatalf("got %+v, filter %q", got, d.filter) @@ -302,21 +304,21 @@ func TestResolveHardwareV1beta2(t *testing.T) { } }) t.Run("a pinned SKU filters discovery and keeps its value", func(t *testing.T) { - d := &fakeDiscoverer{found: DiscoveredHardware{SKU: "b200_sxm", GPUs: 4}} + d := &fakeDiscoverer{found: gpu.GPUInfo{System: "b200_sxm", GPUsPerNode: 4, NodesWithGPUs: 1}} got := DGDRV1beta2Options{Hardware: d}.resolveHardware(context.Background(), admissionv1.Create, nil, spec(v1beta2Ptr("b200_sxm"), nil)) if d.filter != "b200_sxm" || got.sku != "" || got.budget != 4 { t.Fatalf("got %+v, filter %q", got, d.filter) } }) t.Run("a pinned budget keeps its value", func(t *testing.T) { - d := &fakeDiscoverer{found: DiscoveredHardware{SKU: "h200_sxm", GPUs: 64}} + d := &fakeDiscoverer{found: gpu.GPUInfo{System: "h200_sxm", GPUsPerNode: 64, NodesWithGPUs: 1}} got := DGDRV1beta2Options{Hardware: d}.resolveHardware(context.Background(), admissionv1.Create, nil, spec(nil, v1beta2Ptr(int32(3)))) if got.sku != "h200_sxm" || got.budget != 0 { t.Fatalf("got %+v", got) } }) t.Run("the budget is capped", func(t *testing.T) { - d := &fakeDiscoverer{found: DiscoveredHardware{SKU: "h200_sxm", GPUs: 1024}} + d := &fakeDiscoverer{found: gpu.GPUInfo{System: "h200_sxm", GPUsPerNode: 1024, NodesWithGPUs: 1}} if got := (DGDRV1beta2Options{Hardware: d}).resolveHardware(context.Background(), admissionv1.Create, nil, spec(nil, nil)); got.budget != 32 { t.Fatalf("default cap: %+v", got) } @@ -325,7 +327,7 @@ func TestResolveHardwareV1beta2(t *testing.T) { } }) t.Run("an unknown GPU model yields no SKU", func(t *testing.T) { - d := &fakeDiscoverer{found: DiscoveredHardware{GPUs: 8}} + d := &fakeDiscoverer{found: gpu.GPUInfo{GPUsPerNode: 8, NodesWithGPUs: 1}} if got := (DGDRV1beta2Options{Hardware: d}).resolveHardware(context.Background(), admissionv1.Create, nil, spec(nil, nil)); got.sku != "" || got.budget != 8 { t.Fatalf("got %+v", got) } @@ -337,7 +339,7 @@ func TestResolveHardwareV1beta2(t *testing.T) { } }) t.Run("the timeout is configurable", func(t *testing.T) { - d := &fakeDiscoverer{found: DiscoveredHardware{GPUs: 1}} + d := &fakeDiscoverer{found: gpu.GPUInfo{GPUsPerNode: 1, NodesWithGPUs: 1}} DGDRV1beta2Options{Hardware: d, HardwareDiscoveryTimeout: time.Hour}.resolveHardware(context.Background(), admissionv1.Create, nil, spec(nil, nil)) if d.timeout < time.Minute { t.Fatalf("timeout = %v", d.timeout) @@ -346,7 +348,7 @@ func TestResolveHardwareV1beta2(t *testing.T) { } func TestResolveDefaultsV1beta2(t *testing.T) { - d := &fakeDiscoverer{found: DiscoveredHardware{SKU: "h200_sxm", GPUs: 8}} + d := &fakeDiscoverer{found: gpu.GPUInfo{System: "h200_sxm", GPUsPerNode: 8, NodesWithGPUs: 1}} got := DGDRV1beta2Options{Hardware: d, AISimulateImage: "registry.example.com/aisimulate:1.2.0"}. resolveDefaults(context.Background(), admissionv1.Create, nil, v1beta2SchemaDefaulted()) if got.aisimulateImage != "registry.example.com/aisimulate:1.2.0" || got.hardware.sku != "h200_sxm" || got.hardware.budget != 8 { @@ -368,7 +370,7 @@ func v1beta2Envelope(t *testing.T, spec *nvidiacomv1beta2.DynamoGraphDeploymentR func TestDefaultEnvelopeV1beta2(t *testing.T) { options := DGDRV1beta2Options{ AISimulateImage: "registry.example.com/aisimulate:1.2.0", - Hardware: &fakeDiscoverer{found: DiscoveredHardware{SKU: "h200_sxm", GPUs: 8}}, + Hardware: &fakeDiscoverer{found: gpu.GPUInfo{System: "h200_sxm", GPUsPerNode: 8, NodesWithGPUs: 1}}, } dgdr := v1beta2Envelope(t, v1beta2SchemaDefaulted()) @@ -436,7 +438,7 @@ func hardwareSpecV1beta2(sku *string, budget *int32) *nvidiacomv1beta2.DynamoGra func TestHardwareIsNeverDiscoveredOnOtherOperations(t *testing.T) { for _, op := range []admissionv1.Operation{admissionv1.Delete, admissionv1.Connect, ""} { t.Run("operation "+string(op), func(t *testing.T) { - d := &fakeDiscoverer{found: DiscoveredHardware{SKU: "h200_sxm", GPUs: 8}} + d := &fakeDiscoverer{found: gpu.GPUInfo{System: "h200_sxm", GPUsPerNode: 8, NodesWithGPUs: 1}} old := hardwareSpecV1beta2(v1beta2Ptr("h100_sxm"), v1beta2Ptr(int32(4))) got := DGDRV1beta2Options{Hardware: d}.resolveHardware(context.Background(), op, old, hardwareSpecV1beta2(nil, nil)) if got != (hardwareDefaultsV1beta2{}) || d.calls != 0 { @@ -482,7 +484,7 @@ func TestResolveHardwareByOperationV1beta2(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - d := &fakeDiscoverer{found: DiscoveredHardware{SKU: "h200_sxm", GPUs: 8}} + d := &fakeDiscoverer{found: gpu.GPUInfo{System: "h200_sxm", GPUsPerNode: 8, NodesWithGPUs: 1}} got := DGDRV1beta2Options{Hardware: d}.resolveHardware(context.Background(), tt.op, tt.old, tt.spec) if got != tt.want || d.calls != tt.wantCalls || (tt.wantCalls > 0 && d.filter != tt.wantFilter) { t.Fatalf("got %+v, calls %d, filter %q; want %+v, calls %d, filter %q", @@ -495,7 +497,7 @@ func TestResolveHardwareByOperationV1beta2(t *testing.T) { // A Request created while discovery had nothing to offer keeps its spec through a later // metadata-only update, even after the cluster recovers. func TestUpdateKeepsUnchangedHardwareUnresolved(t *testing.T) { - options := DGDRV1beta2Options{Hardware: &fakeDiscoverer{found: DiscoveredHardware{SKU: "h200_sxm", GPUs: 8}}} + options := DGDRV1beta2Options{Hardware: &fakeDiscoverer{found: gpu.GPUInfo{System: "h200_sxm", GPUsPerNode: 8, NodesWithGPUs: 1}}} // The objective metric is still defaulted on update, so the spec already carries it. spec := v1beta2SchemaDefaulted() metric := defaultMetricV1beta2 @@ -512,7 +514,7 @@ func TestUpdateKeepsUnchangedHardwareUnresolved(t *testing.T) { } func TestUpdateRediscoversRemovedHardware(t *testing.T) { - options := DGDRV1beta2Options{Hardware: &fakeDiscoverer{found: DiscoveredHardware{SKU: "h200_sxm", GPUs: 8}}} + options := DGDRV1beta2Options{Hardware: &fakeDiscoverer{found: gpu.GPUInfo{System: "h200_sxm", GPUsPerNode: 8, NodesWithGPUs: 1}}} dgdr := v1beta2Envelope(t, v1beta2SchemaDefaulted()) old := hardwareSpecV1beta2(v1beta2Ptr("h100_sxm"), v1beta2Ptr(int32(4))) if err := defaultDynamoGraphDeploymentRequestV1beta2(context.Background(), dgdr, admissionv1.Update, old, options); err != nil { diff --git a/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_validation_envtest_test.go b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_validation_envtest_test.go index 3c8bd04d4951..e6dac2d48b02 100644 --- a/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_validation_envtest_test.go +++ b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_validation_envtest_test.go @@ -29,7 +29,7 @@ import ( nvidiacomv1beta2 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta2" "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" "github.com/ai-dynamo/dynamo/deploy/operator/internal/features" - webhookdefaulting "github.com/ai-dynamo/dynamo/deploy/operator/internal/webhook/defaulting" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/gpu" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" @@ -60,7 +60,7 @@ func TestDynamoGraphDeploymentRequestValidator_Validate(t *testing.T) { "spec":{"components":[{"name":"VllmPrefillWorker"}]} }` - discoveredH200 := webhookdefaulting.DiscoveredHardware{SKU: "h200_sxm", GPUs: 8} + discoveredH200 := gpu.GPUInfo{System: "h200_sxm", GPUsPerNode: 8, NodesWithGPUs: 1} tests := []struct { name string @@ -70,7 +70,7 @@ func TestDynamoGraphDeploymentRequestValidator_Validate(t *testing.T) { seedWithoutWebhook bool // discovered and discoveryErr are what cluster GPU discovery reports while the case // runs; beforeUpdate may change them between the seed and the update. - discovered webhookdefaulting.DiscoveredHardware + discovered gpu.GPUInfo discoveryErr error beforeUpdate func(*testing.T) // check inspects the object the API server returned. @@ -873,7 +873,7 @@ func TestDynamoGraphDeploymentRequestValidator_Validate(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { admissionHardware.set(tt.discovered, tt.discoveryErr) - t.Cleanup(func() { admissionHardware.set(webhookdefaulting.DiscoveredHardware{}, nil) }) + t.Cleanup(func() { admissionHardware.set(gpu.GPUInfo{}, nil) }) gates := features.Gates{GPUDiscovery: tt.gpuDiscovery} test := admissionTestCase{ diff --git a/deploy/operator/internal/webhook/validation/suite_envtest_test.go b/deploy/operator/internal/webhook/validation/suite_envtest_test.go index 0886a298281e..f39b4e38bc02 100644 --- a/deploy/operator/internal/webhook/validation/suite_envtest_test.go +++ b/deploy/operator/internal/webhook/validation/suite_envtest_test.go @@ -20,6 +20,7 @@ import ( nvidiacomv1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" nvidiacomv1beta2 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta2" "github.com/ai-dynamo/dynamo/deploy/operator/internal/features" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/gpu" "github.com/ai-dynamo/dynamo/deploy/operator/internal/provideroverride" "github.com/ai-dynamo/dynamo/deploy/operator/internal/testing/operatorenv" webhookdefaulting "github.com/ai-dynamo/dynamo/deploy/operator/internal/webhook/defaulting" @@ -106,20 +107,21 @@ func setupAdmissionWebhooks(mgr ctrl.Manager, opts operatorenv.WebhookSetupOptio // so a plain mutex-guarded value is enough. type fakeHardwareDiscoverer struct { mu sync.Mutex - found webhookdefaulting.DiscoveredHardware + found gpu.GPUInfo err error } -func (f *fakeHardwareDiscoverer) set(found webhookdefaulting.DiscoveredHardware, err error) { +func (f *fakeHardwareDiscoverer) set(found gpu.GPUInfo, err error) { f.mu.Lock() defer f.mu.Unlock() f.found, f.err = found, err } -func (f *fakeHardwareDiscoverer) Discover(context.Context, string) (webhookdefaulting.DiscoveredHardware, error) { +func (f *fakeHardwareDiscoverer) Discover(context.Context, nvidiacomv1beta1.GPUSKUType) (*gpu.GPUInfo, error) { f.mu.Lock() defer f.mu.Unlock() - return f.found, f.err + found := f.found + return &found, f.err } type mutableFeatureGate struct { From 0622d43aa756b21b9d65e2a499ae1025d22e03ef Mon Sep 17 00:00:00 2001 From: devivasudevan <49675305+devivasudevan@users.noreply.github.com> Date: Fri, 23 Oct 2026 11:30:25 +0200 Subject: [PATCH 34/40] refactor(operator): leave source-schema rules to the v1beta2 schema and CEL Backend uniqueness is +listType=set and the kvLoadRatio range order is CEL on the type. Native model presence and the model name length were already enforced by the representation CEL and MinLength, so the webhook no longer repeats any of them. The operatorenv table covers the exact error paths. Signed-off-by: devivasudevan <49675305+devivasudevan@users.noreply.github.com> --- .../dynamographdeploymentrequest_types.go | 2 + ...dia.com_dynamographdeploymentrequests.yaml | 4 + .../nvidia.com_dynamographdeploymentruns.yaml | 4 + .../dynamographdeploymentrequest_v1beta2.go | 53 ++------------ ...namographdeploymentrequest_v1beta2_test.go | 73 ++++++------------- ...ploymentrequest_validation_envtest_test.go | 41 +++++++++-- 6 files changed, 71 insertions(+), 106 deletions(-) diff --git a/deploy/operator/api/v1beta2/dynamographdeploymentrequest_types.go b/deploy/operator/api/v1beta2/dynamographdeploymentrequest_types.go index 28a28e4bee2d..37a22bd6c594 100644 --- a/deploy/operator/api/v1beta2/dynamographdeploymentrequest_types.go +++ b/deploy/operator/api/v1beta2/dynamographdeploymentrequest_types.go @@ -70,6 +70,7 @@ type HardwareSpec struct { // +kubebuilder:validation:MaxItems=2 // +kubebuilder:validation:items:Minimum=0 // +kubebuilder:validation:items:Maximum=1 +// +kubebuilder:validation:XValidation:rule="size(self) != 2 || self[0] < self[1]",message="a range needs min < max" type KVLoadRatio []float64 // StaticWorkloadSpec defines one of AISimulate's synthetic workload shapes. @@ -225,6 +226,7 @@ type NativeSearchInput struct { // Backends lists one or more inference backends searched by AISimulate and used by generated candidates. // +optional // +kubebuilder:validation:MinItems=1 + // +listType=set Backends []Backend `json:"backends,omitempty"` // Hardware bounds the accelerator configurations evaluated by the search. diff --git a/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentrequests.yaml b/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentrequests.yaml index 2d143dfd67e5..411e08a3f8c4 100644 --- a/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentrequests.yaml +++ b/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentrequests.yaml @@ -9623,6 +9623,7 @@ spec: type: string minItems: 1 type: array + x-kubernetes-list-type: set hardware: description: |- Hardware bounds the accelerator configurations evaluated by the search. @@ -9791,6 +9792,9 @@ spec: maxItems: 2 minItems: 1 type: array + x-kubernetes-validations: + - message: a range needs min < max + rule: size(self) != 2 || self[0] < self[1] numRequestRatio: description: NumRequestRatio sets request count relative to the selected load. exclusiveMinimum: true diff --git a/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentruns.yaml b/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentruns.yaml index 9a38dec2c50c..bb7ed65cfa81 100644 --- a/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentruns.yaml +++ b/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentruns.yaml @@ -95,6 +95,7 @@ spec: type: string minItems: 1 type: array + x-kubernetes-list-type: set hardware: description: |- Hardware bounds the accelerator configurations evaluated by the search. @@ -255,6 +256,9 @@ spec: maxItems: 2 minItems: 1 type: array + x-kubernetes-validations: + - message: a range needs min < max + rule: size(self) != 2 || self[0] < self[1] numRequestRatio: description: NumRequestRatio sets request count relative to the selected load. exclusiveMinimum: true diff --git a/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_v1beta2.go b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_v1beta2.go index ad26540ca670..0030d86bd34c 100644 --- a/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_v1beta2.go +++ b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_v1beta2.go @@ -19,7 +19,6 @@ package validation import ( "encoding/json" - "strings" nvidiacomv1beta2 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta2" "github.com/ai-dynamo/dynamo/deploy/operator/internal/controller/aisimulate" @@ -51,56 +50,16 @@ func validateDynamoGraphDeploymentRequestV1beta2(spec *nvidiacomv1beta2.DynamoGr return validateDynamoGraphDeploymentRequestSpecV1beta2(spec, field.NewPath("spec")) } -// validateDynamoGraphDeploymentRequestSpecV1beta2 validates spec. spec and fldPath must not be nil. +// validateDynamoGraphDeploymentRequestSpecV1beta2 validates the typed rules that the source +// schema and CEL cannot express. The generated OpenAPI schema and CEL of the v1beta2 API have +// already run before conversion reaches this webhook, so they are not repeated here: native +// model presence, the model name length, backend uniqueness and the kvLoadRatio range order. +// spec and fldPath must not be nil. func validateDynamoGraphDeploymentRequestSpecV1beta2( spec *nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec, fldPath *field.Path, ) field.ErrorList { - var errs field.ErrorList - errs = append(errs, validateModelSpecV1beta2(spec.Model, fldPath.Child("model"))...) - errs = append(errs, validateBackendsV1beta2(spec.Backends, fldPath.Child("backends"))...) - if spec.Workload != nil { - errs = append(errs, validateWorkloadSpecV1beta2(spec.Workload, fldPath.Child("workload"))...) - } - errs = append(errs, validateAISimulateSpecV1beta2(spec.AISimulate, fldPath.Child("aisimulate"))...) - return errs -} - -func validateModelSpecV1beta2(model *nvidiacomv1beta2.ModelSpec, fldPath *field.Path) field.ErrorList { - if model == nil { - return field.ErrorList{field.Required(fldPath, "model is required")} - } - if strings.TrimSpace(model.Name) == "" { - return field.ErrorList{field.Required(fldPath.Child("name"), "must not be blank")} - } - return nil -} - -func validateBackendsV1beta2(backends []nvidiacomv1beta2.Backend, fldPath *field.Path) field.ErrorList { - var errs field.ErrorList - seen := make(map[nvidiacomv1beta2.Backend]bool, len(backends)) - for i, backend := range backends { - if seen[backend] { - errs = append(errs, field.Duplicate(fldPath.Index(i), backend)) - } - seen[backend] = true - } - return errs -} - -func validateWorkloadSpecV1beta2(workload *nvidiacomv1beta2.WorkloadSpec, fldPath *field.Path) field.ErrorList { - if workload.Static == nil { - return nil - } - return validateStaticWorkloadSpecV1beta2(workload.Static, fldPath.Child("static")) -} - -func validateStaticWorkloadSpecV1beta2(static *nvidiacomv1beta2.StaticWorkloadSpec, fldPath *field.Path) field.ErrorList { - // The schema bounds each ratio to [0, 1] and the number of entries to one or two. - if len(static.KVLoadRatio) == 2 && static.KVLoadRatio[0] >= static.KVLoadRatio[1] { - return field.ErrorList{field.Invalid(fldPath.Child("kvLoadRatio"), static.KVLoadRatio, "a range needs min < max")} - } - return nil + return validateAISimulateSpecV1beta2(spec.AISimulate, fldPath.Child("aisimulate")) } // validateAISimulateSpecV1beta2 validates the execution artifact. Admission persists the diff --git a/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_v1beta2_test.go b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_v1beta2_test.go index e0120bcb0827..385599a5dd86 100644 --- a/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_v1beta2_test.go +++ b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_v1beta2_test.go @@ -84,56 +84,25 @@ func TestValidateV1beta2AcceptsADefaultedRequest(t *testing.T) { v1beta2RequireNoError(t, validateDynamoGraphDeploymentRequestV1beta2(v1beta2DefaultedSpec())) } -func TestValidateV1beta2TypedStage(t *testing.T) { - t.Run("missing model", func(t *testing.T) { - spec := v1beta2DefaultedSpec() - spec.Model = nil - errs := validateDynamoGraphDeploymentRequestV1beta2(spec) - v1beta2RequireError(t, errs, "spec.model: model is required") - }) - t.Run("blank model", func(t *testing.T) { - spec := v1beta2DefaultedSpec() - spec.Model.Name = " " - errs := validateDynamoGraphDeploymentRequestV1beta2(spec) - v1beta2RequireError(t, errs, "spec.model.name: must not be blank") - }) - t.Run("duplicate backend", func(t *testing.T) { - spec := v1beta2DefaultedSpec() - spec.Backends = []nvidiacomv1beta2.Backend{nvidiacomv1beta2.BackendVLLM, nvidiacomv1beta2.BackendVLLM} - v1beta2RequireError(t, validateDynamoGraphDeploymentRequestV1beta2(spec), "spec.backends[1]") - }) - t.Run("kv_load_ratio range order", func(t *testing.T) { - spec := v1beta2DefaultedSpec() - spec.Workload.Static.KVLoadRatio = nvidiacomv1beta2.KVLoadRatio{0.9, 0.1} - v1beta2RequireError(t, validateDynamoGraphDeploymentRequestV1beta2(spec), "spec.workload.static.kvLoadRatio: a range needs min < max") - }) - t.Run("an empty workload has nothing to validate", func(t *testing.T) { - spec := v1beta2WithParameters(v1beta2DefaultedSpec(), `{"workload":{"isl":512,"osl":64,"concurrency":8}}`) - spec.Workload = &nvidiacomv1beta2.WorkloadSpec{} - v1beta2RequireNoError(t, validateDynamoGraphDeploymentRequestV1beta2(spec)) - }) - t.Run("an invalid typed value is not excused by the parameters", func(t *testing.T) { - spec := v1beta2WithParameters(v1beta2DefaultedSpec(), `{"workload":{"kv_load_ratio":0.5}}`) - spec.Workload.Static.KVLoadRatio = nvidiacomv1beta2.KVLoadRatio{0.9, 0.1} - v1beta2RequireError(t, validateDynamoGraphDeploymentRequestV1beta2(spec), "spec.workload.static.kvLoadRatio") - }) - t.Run("typed errors aggregate", func(t *testing.T) { - spec := v1beta2DefaultedSpec() - spec.Model.Name = "" - spec.Backends = []nvidiacomv1beta2.Backend{nvidiacomv1beta2.BackendVLLM, nvidiacomv1beta2.BackendVLLM} - errs := validateDynamoGraphDeploymentRequestV1beta2(spec) - v1beta2RequireError(t, errs, "spec.model.name") - v1beta2RequireError(t, errs, "spec.backends[1]") - }) - t.Run("parameters are opaque to admission", func(t *testing.T) { - for _, parameters := range []string{ - `{"workload":{"isl":0},"goal":{"target":"nope"}}`, - `[1]`, // the CRD types parameters as an object; admission does not interpret it - } { - v1beta2RequireNoError(t, validateDynamoGraphDeploymentRequestV1beta2( - v1beta2WithParameters(v1beta2DefaultedSpec(), parameters))) - } - }) +// The webhook does not repeat what the generated schema and CEL enforce before conversion: +// native model presence, the model name length, backend uniqueness and the kvLoadRatio range +// order. The operatorenv table proves those errors end to end. +func TestValidateV1beta2LeavesSourceSchemaRulesToTheSchema(t *testing.T) { + spec := v1beta2DefaultedSpec() + spec.Model = nil + spec.Backends = []nvidiacomv1beta2.Backend{nvidiacomv1beta2.BackendVLLM, nvidiacomv1beta2.BackendVLLM} + spec.Workload.Static.KVLoadRatio = nvidiacomv1beta2.KVLoadRatio{0.9, 0.1} + v1beta2RequireNoError(t, validateDynamoGraphDeploymentRequestV1beta2(spec)) +} + +func TestValidateV1beta2ParametersAreOpaque(t *testing.T) { + for _, parameters := range []string{ + `{"workload":{"isl":0},"goal":{"target":"nope"}}`, + `[1]`, // the CRD types parameters as an object; admission does not interpret it + } { + v1beta2RequireNoError(t, validateDynamoGraphDeploymentRequestV1beta2( + v1beta2WithParameters(v1beta2DefaultedSpec(), parameters))) + } } func TestValidateV1beta2AISimulateImage(t *testing.T) { @@ -174,9 +143,9 @@ func TestValidateEnvelopeV1beta2(t *testing.T) { v1beta2RequireNoError(t, validateDynamoGraphDeploymentRequestEnvelopeV1beta2(v1beta2Envelope(string(good)), path)) bad := v1beta2DefaultedSpec() - bad.Model.Name = "" + bad.AISimulate = nil raw, _ := json.Marshal(bad) - v1beta2RequireError(t, validateDynamoGraphDeploymentRequestEnvelopeV1beta2(v1beta2Envelope(string(raw)), path), "model") + v1beta2RequireError(t, validateDynamoGraphDeploymentRequestEnvelopeV1beta2(v1beta2Envelope(string(raw)), path), "aisimulate") v1beta2RequireError(t, validateDynamoGraphDeploymentRequestEnvelopeV1beta2(v1beta2Envelope(`{"model": 3}`), path), "spec.v1beta2: must be a valid v1beta2 spec") } diff --git a/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_validation_envtest_test.go b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_validation_envtest_test.go index e6dac2d48b02..2be3a73007b6 100644 --- a/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_validation_envtest_test.go +++ b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_validation_envtest_test.go @@ -742,24 +742,51 @@ func TestDynamoGraphDeploymentRequestValidator_Validate(t *testing.T) { { name: "v1beta2 stored invalid spec is rejected again on a metadata-only update", oldRequest: v1beta2DGDRForAdmission(func(spec map[string]any) { - spec["backends"] = []any{"vllm", "vllm"} + spec["aisimulate"] = map[string]any{"image": "registry.example.com/aisimulate:custom"} }, nil), request: v1beta2DGDRForAdmission(func(spec map[string]any) { - spec["backends"] = []any{"vllm", "vllm"} + spec["aisimulate"] = map[string]any{"image": "registry.example.com/aisimulate:custom"} }, map[string]any{"updated": "true"}), seedWithoutWebhook: true, gpuDiscovery: true, discovered: discoveredH200, - wantWebhook: []string{`spec.backends[1]: Duplicate value: "vllm"`}, + wantWebhook: []string{ + "spec.aisimulate.versionOverride: Required value: is required when spec.aisimulate.image has no parseable version tag", + }, }, { name: "v1beta2 duplicate backends are rejected on create", request: v1beta2DGDRForAdmission(func(spec map[string]any) { spec["backends"] = []any{"vllm", "vllm"} }, nil), + gpuDiscovery: true, + discovered: discoveredH200, + wantSchemaErr: `spec.backends[1]: Duplicate value: "vllm"`, + }, + { + name: "v1beta2 missing model is rejected by the representation CEL", + request: v1beta2DGDRForAdmission(func(spec map[string]any) { delete(spec, "model") }, nil), + gpuDiscovery: true, + discovered: discoveredH200, + wantCELErr: `spec: Invalid value: exactly one of native v1beta2 fields or v1beta1 must be specified`, + }, + { + name: "v1beta2 empty model name is rejected by the schema", + request: v1beta2DGDRForAdmission(func(spec map[string]any) { + spec["model"] = map[string]any{"name": ""} + }, nil), + gpuDiscovery: true, + discovered: discoveredH200, + wantSchemaErr: `spec.model.name: Invalid value: "": spec.model.name in body should be at least 1 chars long`, + }, + { + name: "v1beta2 kvLoadRatio range needs min below max", + request: v1beta2DGDRForAdmission(func(spec map[string]any) { + spec["workload"] = map[string]any{"static": map[string]any{"kvLoadRatio": []any{0.9, 0.1}}} + }, nil), gpuDiscovery: true, discovered: discoveredH200, - wantWebhook: []string{`spec.backends[1]: Duplicate value: "vllm"`}, + wantCELErr: `spec.workload.static.kvLoadRatio: Invalid value: a range needs min < max`, }, { name: "v1beta2 versionOverride latest is rejected by the schema", @@ -864,9 +891,9 @@ func TestDynamoGraphDeploymentRequestValidator_Validate(t *testing.T) { request: v1beta2DGDRForAdmission(func(spec map[string]any) { spec["backends"] = []any{"sglang", "sglang"} }, nil), - gpuDiscovery: true, - discovered: discoveredH200, - wantWebhook: []string{`spec.backends[1]: Duplicate value: "sglang"`}, + gpuDiscovery: true, + discovered: discoveredH200, + wantSchemaErr: `spec.backends[1]: Duplicate value: "sglang"`, }, } From fba802bdf7fbff79529a704ef7f6487cf9f76b3d Mon Sep 17 00:00:00 2001 From: devivasudevan <49675305+devivasudevan@users.noreply.github.com> Date: Fri, 23 Oct 2026 11:35:54 +0200 Subject: [PATCH 35/40] test(operator): cover v1beta2 defaulting through the operatorenv admission table Move the externally observable defaulting cases into the admission table and delete v1beta2SchemaDefaulted and the direct-function matrix. The unit tests that remain cover resolver and envelope internals. Signed-off-by: devivasudevan <49675305+devivasudevan@users.noreply.github.com> --- ...ploymentrequest_v1beta2_structural_test.go | 157 -------- ...namographdeploymentrequest_v1beta2_test.go | 339 ++---------------- ...ploymentrequest_validation_envtest_test.go | 92 ++++- 3 files changed, 116 insertions(+), 472 deletions(-) delete mode 100644 deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_structural_test.go diff --git a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_structural_test.go b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_structural_test.go deleted file mode 100644 index 270e3732a5a1..000000000000 --- a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_structural_test.go +++ /dev/null @@ -1,157 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package defaulting - -import ( - "encoding/json" - "testing" - - nvidiacomv1beta2 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta2" - "k8s.io/utils/ptr" -) - -func structuralTestOptions() dynamoGraphDeploymentRequestV1beta2DefaultOptions { - return dynamoGraphDeploymentRequestV1beta2DefaultOptions{ - hardware: hardwareDefaultsV1beta2{sku: "h200_sxm", budget: 8}, - aisimulateImage: "planner:test", - } -} - -func structuralDefaultedSpec() nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec { - return nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec{ - Backends: []nvidiacomv1beta2.Backend{"vllm", "sglang", "trtllm"}, - Hardware: &nvidiacomv1beta2.HardwareSpec{GPU: &nvidiacomv1beta2.GPUHardwareSpec{ - SKU: ptr.To("h200_sxm"), Budget: ptr.To(int32(8)), - }}, - Workload: &nvidiacomv1beta2.WorkloadSpec{Static: &nvidiacomv1beta2.StaticWorkloadSpec{ - ISL: ptr.To(int32(1024)), OSL: ptr.To(int32(128)), Concurrency: ptr.To(int32(10)), - }}, - Objective: &nvidiacomv1beta2.ObjectiveSpec{ - Mode: nvidiacomv1beta2.ObjectiveModeOptimize, - Metric: ptr.To(nvidiacomv1beta2.ObjectiveMetric("Throughput")), - }, - AISimulate: &nvidiacomv1beta2.AISimulateSpec{Image: "planner:test"}, - Recommendation: &nvidiacomv1beta2.RecommendationSpec{MaxCandidates: 1}, - } -} - -func mustJSON(t *testing.T, v any) string { - t.Helper() - raw, err := json.Marshal(v) - if err != nil { - t.Fatal(err) - } - return string(raw) -} - -func TestStructuralDefaultsV1beta2(t *testing.T) { - tests := []struct { - name string - input nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec - want func() nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec - }{ - { - name: "an empty spec receives every default", - input: nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec{}, - want: structuralDefaultedSpec, - }, - { - name: "empty parents receive their own defaults", - input: nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec{ - Workload: &nvidiacomv1beta2.WorkloadSpec{}, - Objective: &nvidiacomv1beta2.ObjectiveSpec{}, - Recommendation: &nvidiacomv1beta2.RecommendationSpec{}, - }, - want: structuralDefaultedSpec, - }, - { - name: "workload.static: {} is an opt-out and stays empty", - input: nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec{ - Workload: &nvidiacomv1beta2.WorkloadSpec{Static: &nvidiacomv1beta2.StaticWorkloadSpec{}}, - }, - want: func() nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec { - spec := structuralDefaultedSpec() - spec.Workload = &nvidiacomv1beta2.WorkloadSpec{Static: &nvidiacomv1beta2.StaticWorkloadSpec{}} - return spec - }, - }, - { - name: "explicit values are kept", - input: nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec{ - Backends: []nvidiacomv1beta2.Backend{"vllm"}, - Hardware: &nvidiacomv1beta2.HardwareSpec{GPU: &nvidiacomv1beta2.GPUHardwareSpec{ - SKU: ptr.To("h100_sxm"), Budget: ptr.To(int32(4)), - }}, - Workload: &nvidiacomv1beta2.WorkloadSpec{Static: &nvidiacomv1beta2.StaticWorkloadSpec{ - ISL: ptr.To(int32(10)), - }}, - Objective: &nvidiacomv1beta2.ObjectiveSpec{ - Mode: nvidiacomv1beta2.ObjectiveModeOptimize, - Metric: ptr.To(nvidiacomv1beta2.ObjectiveMetric("TTFT")), - }, - AISimulate: &nvidiacomv1beta2.AISimulateSpec{Image: "mine:1"}, - Recommendation: &nvidiacomv1beta2.RecommendationSpec{MaxCandidates: 3}, - }, - want: func() nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec { - return nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec{ - Backends: []nvidiacomv1beta2.Backend{"vllm"}, - Hardware: &nvidiacomv1beta2.HardwareSpec{GPU: &nvidiacomv1beta2.GPUHardwareSpec{ - SKU: ptr.To("h100_sxm"), Budget: ptr.To(int32(4)), - }}, - Workload: &nvidiacomv1beta2.WorkloadSpec{Static: &nvidiacomv1beta2.StaticWorkloadSpec{ - ISL: ptr.To(int32(10)), - }}, - Objective: &nvidiacomv1beta2.ObjectiveSpec{ - Mode: nvidiacomv1beta2.ObjectiveModeOptimize, - Metric: ptr.To(nvidiacomv1beta2.ObjectiveMetric("TTFT")), - }, - AISimulate: &nvidiacomv1beta2.AISimulateSpec{Image: "mine:1"}, - Recommendation: &nvidiacomv1beta2.RecommendationSpec{MaxCandidates: 3}, - } - }, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - spec := tt.input - defaultDynamoGraphDeploymentRequestSpecV1beta2(&spec, structuralTestOptions()) - want := tt.want() - if got, wantJSON := mustJSON(t, &spec), mustJSON(t, &want); got != wantJSON { - t.Fatalf("defaulted spec mismatch\n got: %s\nwant: %s", got, wantJSON) - } - - // Defaulting is idempotent. - again := spec - defaultDynamoGraphDeploymentRequestSpecV1beta2(&again, structuralTestOptions()) - if got, wantJSON := mustJSON(t, &again), mustJSON(t, &spec); got != wantJSON { - t.Fatalf("second pass changed the spec\n got: %s\nwant: %s", got, wantJSON) - } - }) - } -} - -func TestStructuralDefaultsV1beta2WithoutEnvironment(t *testing.T) { - var spec nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec - defaultDynamoGraphDeploymentRequestSpecV1beta2(&spec, dynamoGraphDeploymentRequestV1beta2DefaultOptions{}) - if spec.Hardware != nil || spec.AISimulate != nil { - t.Fatalf("hardware and aisimulate must stay absent without an environment: %s", mustJSON(t, &spec)) - } - if spec.Workload == nil || spec.Objective == nil || spec.Recommendation == nil || spec.Backends == nil { - t.Fatalf("static defaults must still apply: %s", mustJSON(t, &spec)) - } -} diff --git a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_test.go b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_test.go index e02001a815cc..1f8c0b9aa77e 100644 --- a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_test.go +++ b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_test.go @@ -26,234 +26,13 @@ import ( nvidiacomv1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" nvidiacomv1beta2 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta2" - "github.com/ai-dynamo/dynamo/deploy/operator/internal/controller/aisimulate" "github.com/ai-dynamo/dynamo/deploy/operator/internal/gpu" admissionv1 "k8s.io/api/admission/v1" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" - "k8s.io/apimachinery/pkg/runtime" ) func v1beta2Ptr[T any](v T) *T { return &v } -// v1beta2SchemaDefaulted is a model-only Request as the API server hands it to admission: the -// CRD has applied its defaults, and admission has not run. -func v1beta2SchemaDefaulted() *nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec { - return &nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec{ - Model: &nvidiacomv1beta2.ModelSpec{Name: "meta-llama/Llama-3.1-8B-Instruct"}, - Backends: []nvidiacomv1beta2.Backend{nvidiacomv1beta2.BackendVLLM, nvidiacomv1beta2.BackendSGLang, nvidiacomv1beta2.BackendTRTLLM}, - Workload: &nvidiacomv1beta2.WorkloadSpec{Static: &nvidiacomv1beta2.StaticWorkloadSpec{ - ISL: v1beta2Ptr(int32(1024)), OSL: v1beta2Ptr(int32(128)), Concurrency: v1beta2Ptr(int32(10)), - }}, - Objective: &nvidiacomv1beta2.ObjectiveSpec{ - Mode: nvidiacomv1beta2.ObjectiveModeOptimize, - }, - Recommendation: &nvidiacomv1beta2.RecommendationSpec{MaxCandidates: 1}, - } -} - -func v1beta2JSON(t *testing.T, spec *nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec) string { - t.Helper() - raw, err := json.Marshal(spec) - if err != nil { - t.Fatal(err) - } - return string(raw) -} - -func TestDefaultSpecV1beta2ModelOnlyRequest(t *testing.T) { - spec := v1beta2SchemaDefaulted() - defaultDynamoGraphDeploymentRequestSpecV1beta2(spec, dynamoGraphDeploymentRequestV1beta2DefaultOptions{ - hardware: hardwareDefaultsV1beta2{sku: "h200_sxm", budget: 8}, - aisimulateImage: "registry.example.com/aisimulate:1.2.0", - }) - if got := ptrDeref(spec.Objective.Metric); got != "Throughput" { - t.Fatalf("metric = %q", got) - } - gpu := spec.Hardware.GPU - if *gpu.SKU != "h200_sxm" || *gpu.Budget != 8 { - t.Fatalf("hardware = %s/%d", *gpu.SKU, *gpu.Budget) - } - if spec.AISimulate.Image != "registry.example.com/aisimulate:1.2.0" { - t.Fatalf("image = %q", spec.AISimulate.Image) - } -} - -func ptrDeref(metric *nvidiacomv1beta2.ObjectiveMetric) nvidiacomv1beta2.ObjectiveMetric { - if metric == nil { - return "" - } - return *metric -} - -func TestDefaultSpecV1beta2IsIdempotent(t *testing.T) { - opts := dynamoGraphDeploymentRequestV1beta2DefaultOptions{ - hardware: hardwareDefaultsV1beta2{sku: "h200_sxm", budget: 8}, - aisimulateImage: "registry.example.com/aisimulate:1.2.0", - } - spec := v1beta2SchemaDefaulted() - defaultDynamoGraphDeploymentRequestSpecV1beta2(spec, opts) - first := v1beta2JSON(t, spec) - defaultDynamoGraphDeploymentRequestSpecV1beta2(spec, opts) - if second := v1beta2JSON(t, spec); second != first { - t.Fatalf("a second pass changed the spec:\n%s\n%s", first, second) - } -} - -func TestDefaultSpecV1beta2PreservesExplicitValues(t *testing.T) { - opts := dynamoGraphDeploymentRequestV1beta2DefaultOptions{ - hardware: hardwareDefaultsV1beta2{sku: "h200_sxm", budget: 8}, - aisimulateImage: "registry.example.com/aisimulate:1.2.0", - } - t.Run("hardware", func(t *testing.T) { - spec := v1beta2SchemaDefaulted() - spec.Hardware = &nvidiacomv1beta2.HardwareSpec{GPU: &nvidiacomv1beta2.GPUHardwareSpec{SKU: v1beta2Ptr("b200_sxm"), Budget: v1beta2Ptr(int32(2))}} - defaultDynamoGraphDeploymentRequestSpecV1beta2(spec, opts) - if *spec.Hardware.GPU.SKU != "b200_sxm" || *spec.Hardware.GPU.Budget != 2 { - t.Fatalf("hardware overwritten: %s/%d", *spec.Hardware.GPU.SKU, *spec.Hardware.GPU.Budget) - } - }) - t.Run("a pinned SKU lets admission size the budget", func(t *testing.T) { - spec := v1beta2SchemaDefaulted() - spec.Hardware = &nvidiacomv1beta2.HardwareSpec{GPU: &nvidiacomv1beta2.GPUHardwareSpec{SKU: v1beta2Ptr("b200_sxm")}} - defaultDynamoGraphDeploymentRequestSpecV1beta2(spec, opts) - if *spec.Hardware.GPU.SKU != "b200_sxm" || *spec.Hardware.GPU.Budget != 8 { - t.Fatalf("got %s/%d", *spec.Hardware.GPU.SKU, *spec.Hardware.GPU.Budget) - } - }) - t.Run("an empty hardware object is filled", func(t *testing.T) { - spec := v1beta2SchemaDefaulted() - spec.Hardware = &nvidiacomv1beta2.HardwareSpec{} - defaultDynamoGraphDeploymentRequestSpecV1beta2(spec, opts) - if spec.Hardware.GPU == nil || *spec.Hardware.GPU.SKU != "h200_sxm" { - t.Fatal("hardware not filled") - } - }) - t.Run("metric", func(t *testing.T) { - spec := v1beta2SchemaDefaulted() - spec.Objective.Metric = v1beta2Ptr(nvidiacomv1beta2.ObjectiveMetric("TTFT")) - defaultDynamoGraphDeploymentRequestSpecV1beta2(spec, opts) - if *spec.Objective.Metric != "TTFT" { - t.Fatalf("metric overwritten: %s", *spec.Objective.Metric) - } - }) - t.Run("image", func(t *testing.T) { - spec := v1beta2SchemaDefaulted() - spec.AISimulate = &nvidiacomv1beta2.AISimulateSpec{Image: "registry.example.com/aisimulate:custom", VersionOverride: "1.4.0"} - defaultDynamoGraphDeploymentRequestSpecV1beta2(spec, opts) - if spec.AISimulate.Image != "registry.example.com/aisimulate:custom" { - t.Fatalf("image overwritten: %s", spec.AISimulate.Image) - } - }) - t.Run("parameters are untouched", func(t *testing.T) { - spec := v1beta2SchemaDefaulted() - spec.AISimulate = &nvidiacomv1beta2.AISimulateSpec{Parameters: &runtime.RawExtension{Raw: []byte(`{"goal":{"target":"ttft"}}`)}} - defaultDynamoGraphDeploymentRequestSpecV1beta2(spec, opts) - if string(spec.AISimulate.Parameters.Raw) != `{"goal":{"target":"ttft"}}` { - t.Fatal("parameters changed") - } - if spec.AISimulate.Image != "registry.example.com/aisimulate:1.2.0" { - t.Fatal("image not persisted next to parameters") - } - }) -} - -func TestDefaultSpecV1beta2KeepsExplicitValues(t *testing.T) { - opts := dynamoGraphDeploymentRequestV1beta2DefaultOptions{ - hardware: hardwareDefaultsV1beta2{sku: "h200_sxm", budget: 8}, - aisimulateImage: "registry.example.com/aisimulate:1.2.0", - } - t.Run("an empty workload receives the static default", func(t *testing.T) { - spec := v1beta2SchemaDefaulted() - spec.Workload = &nvidiacomv1beta2.WorkloadSpec{} - defaultDynamoGraphDeploymentRequestSpecV1beta2(spec, opts) - if !hasModelOnlyStatic(spec.Workload) { - t.Fatal("workload: {} must receive the static model-only default") - } - }) - t.Run("an explicit empty static workload is an opt-out", func(t *testing.T) { - spec := v1beta2SchemaDefaulted() - spec.Workload = &nvidiacomv1beta2.WorkloadSpec{Static: &nvidiacomv1beta2.StaticWorkloadSpec{}} - defaultDynamoGraphDeploymentRequestSpecV1beta2(spec, opts) - if spec.Workload.Static.ISL != nil || spec.Workload.Static.OSL != nil || spec.Workload.Static.Concurrency != nil { - t.Fatal("an explicit workload.static: {} was filled") - } - }) - t.Run("omitted workload", func(t *testing.T) { - spec := v1beta2SchemaDefaulted() - spec.Workload = nil - defaultDynamoGraphDeploymentRequestSpecV1beta2(spec, opts) - if !hasModelOnlyStatic(spec.Workload) { - t.Fatal("an omitted workload must become workload: {}, with the static model-only default") - } - }) - t.Run("explicit empty SLA", func(t *testing.T) { - spec := v1beta2SchemaDefaulted() - spec.Objective.SLA = &nvidiacomv1beta2.ObjectiveSLASpec{} - defaultDynamoGraphDeploymentRequestSpecV1beta2(spec, opts) - if spec.Objective.SLA.TTFTMs != nil || spec.Objective.SLA.ITLMs != nil { - t.Fatal("an explicit sla: {} was filled") - } - }) - t.Run("an explicit zero-valued backend list element is kept", func(t *testing.T) { - spec := v1beta2SchemaDefaulted() - spec.Backends = []nvidiacomv1beta2.Backend{nvidiacomv1beta2.BackendVLLM} - defaultDynamoGraphDeploymentRequestSpecV1beta2(spec, opts) - if len(spec.Backends) != 1 { - t.Fatal("backends changed") - } - }) -} - -func TestDefaultObjectiveSpecV1beta2(t *testing.T) { - cases := []struct { - name string - objective nvidiacomv1beta2.ObjectiveSpec - want nvidiacomv1beta2.ObjectiveMetric - }{ - {"optimize without a metric", nvidiacomv1beta2.ObjectiveSpec{Mode: nvidiacomv1beta2.ObjectiveModeOptimize}, "Throughput"}, - {"an explicit metric is kept", nvidiacomv1beta2.ObjectiveSpec{Mode: nvidiacomv1beta2.ObjectiveModeOptimize, Metric: v1beta2Ptr(nvidiacomv1beta2.ObjectiveMetric("TTFT"))}, "TTFT"}, - {"pareto has no scalar metric", nvidiacomv1beta2.ObjectiveSpec{Mode: nvidiacomv1beta2.ObjectiveModePareto, Metrics: []nvidiacomv1beta2.ObjectiveMetric{"TTFT", "Throughput"}}, ""}, - {"an absent mode is Optimize, which admission fills", nvidiacomv1beta2.ObjectiveSpec{}, "Throughput"}, - } - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - defaultObjectiveSpecV1beta2(&tc.objective) - if got := ptrDeref(tc.objective.Metric); got != tc.want { - t.Fatalf("metric = %q, want %q", got, tc.want) - } - }) - } -} - -func TestDefaultSpecV1beta2DoesNotCreateWhatTheEnvironmentCannotFill(t *testing.T) { - spec := v1beta2SchemaDefaulted() - defaultDynamoGraphDeploymentRequestSpecV1beta2(spec, dynamoGraphDeploymentRequestV1beta2DefaultOptions{}) - if spec.Hardware != nil || spec.AISimulate != nil { - t.Fatalf("created sections with nothing to put in them: %s", v1beta2JSON(t, spec)) - } - spec.Objective = nil - defaultDynamoGraphDeploymentRequestSpecV1beta2(spec, dynamoGraphDeploymentRequestV1beta2DefaultOptions{}) - if spec.Objective == nil || ptrDeref(spec.Objective.Metric) != "Throughput" { - t.Fatal("an omitted objective must become objective: {} with the Optimize metric default") - } -} - -func TestNativeTargetOverridesTheDefaultedMetric(t *testing.T) { - spec := v1beta2SchemaDefaulted() - spec.AISimulate = &nvidiacomv1beta2.AISimulateSpec{Parameters: &runtime.RawExtension{Raw: []byte(`{"goal":{"target":"ttft"}}`)}} - defaultDynamoGraphDeploymentRequestSpecV1beta2(spec, dynamoGraphDeploymentRequestV1beta2DefaultOptions{}) - if *spec.Objective.Metric != "Throughput" { - t.Fatalf("the typed metric must be defaulted regardless of the parameters, got %q", *spec.Objective.Metric) - } - input, err := aisimulate.Lower(spec) - if err != nil { - t.Fatal(err) - } - if target, _ := input["goal"].(map[string]any)["target"]; target != "ttft" { - t.Fatalf("the native target must win when lowered, got %v", target) - } -} - type fakeDiscoverer struct { found gpu.GPUInfo err error @@ -274,7 +53,7 @@ func (f *fakeDiscoverer) Discover(ctx context.Context, filterSKU nvidiacomv1beta func TestResolveHardwareV1beta2(t *testing.T) { spec := func(sku *string, budget *int32) *nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec { - s := v1beta2SchemaDefaulted() + s := modelOnlySpecV1beta2() if sku != nil || budget != nil { s.Hardware = &nvidiacomv1beta2.HardwareSpec{GPU: &nvidiacomv1beta2.GPUHardwareSpec{SKU: sku, Budget: budget}} } @@ -350,7 +129,7 @@ func TestResolveHardwareV1beta2(t *testing.T) { func TestResolveDefaultsV1beta2(t *testing.T) { d := &fakeDiscoverer{found: gpu.GPUInfo{System: "h200_sxm", GPUsPerNode: 8, NodesWithGPUs: 1}} got := DGDRV1beta2Options{Hardware: d, AISimulateImage: "registry.example.com/aisimulate:1.2.0"}. - resolveDefaults(context.Background(), admissionv1.Create, nil, v1beta2SchemaDefaulted()) + resolveDefaults(context.Background(), admissionv1.Create, nil, modelOnlySpecV1beta2()) if got.aisimulateImage != "registry.example.com/aisimulate:1.2.0" || got.hardware.sku != "h200_sxm" || got.hardware.budget != 8 { t.Fatalf("got %+v", got) } @@ -372,7 +151,7 @@ func TestDefaultEnvelopeV1beta2(t *testing.T) { AISimulateImage: "registry.example.com/aisimulate:1.2.0", Hardware: &fakeDiscoverer{found: gpu.GPUInfo{System: "h200_sxm", GPUsPerNode: 8, NodesWithGPUs: 1}}, } - dgdr := v1beta2Envelope(t, v1beta2SchemaDefaulted()) + dgdr := v1beta2Envelope(t, modelOnlySpecV1beta2()) if err := defaultDynamoGraphDeploymentRequestV1beta2(context.Background(), dgdr, admissionv1.Create, nil, options); err != nil { t.Fatal(err) @@ -401,24 +180,6 @@ func TestDefaultEnvelopeV1beta2(t *testing.T) { } } -func TestDefaultEnvelopeV1beta2LeavesUnchangedEnvelopeUntouched(t *testing.T) { - // Nothing for admission to add: the objective already carries its metric and there is no - // environment to contribute. The envelope bytes stay exactly as sent, even though they are - // not in canonical form. - spec := v1beta2SchemaDefaulted() - spec.Objective.Metric = v1beta2Ptr(nvidiacomv1beta2.ObjectiveMetric("ThroughputPerGPU")) - raw, _ := json.Marshal(spec) - dgdr := v1beta2Envelope(t, spec) - dgdr.Spec.V1Beta2.Raw = append([]byte(" "), raw...) - original := string(dgdr.Spec.V1Beta2.Raw) - if err := defaultDynamoGraphDeploymentRequestV1beta2(context.Background(), dgdr, admissionv1.Create, nil, DGDRV1beta2Options{}); err != nil { - t.Fatal(err) - } - if string(dgdr.Spec.V1Beta2.Raw) != original { - t.Fatalf("envelope was rewritten without a change: %q", dgdr.Spec.V1Beta2.Raw) - } -} - func TestDefaultEnvelopeV1beta2RejectsMalformedSpec(t *testing.T) { dgdr := &nvidiacomv1beta1.DynamoGraphDeploymentRequest{} dgdr.Spec.V1Beta2 = &apiextensionsv1.JSON{Raw: []byte(`{"model": 3}`)} @@ -428,7 +189,7 @@ func TestDefaultEnvelopeV1beta2RejectsMalformedSpec(t *testing.T) { } func hardwareSpecV1beta2(sku *string, budget *int32) *nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec { - spec := v1beta2SchemaDefaulted() + spec := modelOnlySpecV1beta2() if sku != nil || budget != nil { spec.Hardware = &nvidiacomv1beta2.HardwareSpec{GPU: &nvidiacomv1beta2.GPUHardwareSpec{SKU: sku, Budget: budget}} } @@ -494,41 +255,6 @@ func TestResolveHardwareByOperationV1beta2(t *testing.T) { } } -// A Request created while discovery had nothing to offer keeps its spec through a later -// metadata-only update, even after the cluster recovers. -func TestUpdateKeepsUnchangedHardwareUnresolved(t *testing.T) { - options := DGDRV1beta2Options{Hardware: &fakeDiscoverer{found: gpu.GPUInfo{System: "h200_sxm", GPUsPerNode: 8, NodesWithGPUs: 1}}} - // The objective metric is still defaulted on update, so the spec already carries it. - spec := v1beta2SchemaDefaulted() - metric := defaultMetricV1beta2 - spec.Objective.Metric = &metric - dgdr := v1beta2Envelope(t, spec) - first := string(dgdr.Spec.V1Beta2.Raw) - if err := defaultDynamoGraphDeploymentRequestV1beta2( - context.Background(), dgdr, admissionv1.Update, hardwareSpecV1beta2(nil, nil), options); err != nil { - t.Fatal(err) - } - if string(dgdr.Spec.V1Beta2.Raw) != first { - t.Fatalf("update changed the spec:\n%s\n%s", first, dgdr.Spec.V1Beta2.Raw) - } -} - -func TestUpdateRediscoversRemovedHardware(t *testing.T) { - options := DGDRV1beta2Options{Hardware: &fakeDiscoverer{found: gpu.GPUInfo{System: "h200_sxm", GPUsPerNode: 8, NodesWithGPUs: 1}}} - dgdr := v1beta2Envelope(t, v1beta2SchemaDefaulted()) - old := hardwareSpecV1beta2(v1beta2Ptr("h100_sxm"), v1beta2Ptr(int32(4))) - if err := defaultDynamoGraphDeploymentRequestV1beta2(context.Background(), dgdr, admissionv1.Update, old, options); err != nil { - t.Fatal(err) - } - var got nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec - if err := json.Unmarshal(dgdr.Spec.V1Beta2.Raw, &got); err != nil { - t.Fatal(err) - } - if got.Hardware == nil || got.Hardware.GPU == nil || *got.Hardware.GPU.SKU != "h200_sxm" || *got.Hardware.GPU.Budget != 8 { - t.Fatalf("hardware not rediscovered: %+v", got.Hardware) - } -} - func TestDecodeOldSpecV1beta2(t *testing.T) { old, err := json.Marshal(v1beta2Envelope(t, hardwareSpecV1beta2(v1beta2Ptr("h100_sxm"), nil))) if err != nil { @@ -549,39 +275,30 @@ func TestDecodeOldSpecV1beta2(t *testing.T) { } } -func TestDefaultSpecV1beta2StructuralDefaults(t *testing.T) { - opts := dynamoGraphDeploymentRequestV1beta2DefaultOptions{} - - t.Run("an empty native spec receives the four structural parents", func(t *testing.T) { - spec := &nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec{} - defaultDynamoGraphDeploymentRequestSpecV1beta2(spec, opts) - if len(spec.Backends) != 3 { - t.Fatalf("backends = %v, want vllm, sglang and trtllm", spec.Backends) - } - if !hasModelOnlyStatic(spec.Workload) { - t.Fatalf("workload = %s, want {} with the static model-only default", v1beta2JSON(t, spec)) - } - if spec.Objective == nil || spec.Recommendation == nil { - t.Fatalf("objective and recommendation must exist: %s", v1beta2JSON(t, spec)) - } - }) - t.Run("explicit values and opt-outs are kept", func(t *testing.T) { - spec := &nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec{} - spec.Backends = []nvidiacomv1beta2.Backend{nvidiacomv1beta2.BackendVLLM} - spec.Workload = &nvidiacomv1beta2.WorkloadSpec{Static: &nvidiacomv1beta2.StaticWorkloadSpec{}} - defaultDynamoGraphDeploymentRequestSpecV1beta2(spec, opts) - if len(spec.Backends) != 1 { - t.Fatalf("backends changed: %v", spec.Backends) - } - if spec.Workload.Static == nil { - t.Fatal("workload.static: {} (the opt-out) was removed") - } - }) +// modelOnlySpecV1beta2 is a Request as a client sends it: only the model is named. +func modelOnlySpecV1beta2() *nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec { + spec := &nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec{} + spec.Model = &nvidiacomv1beta2.ModelSpec{Name: "meta-llama/Llama-3.1-8B-Instruct"} + return spec } -func hasModelOnlyStatic(w *nvidiacomv1beta2.WorkloadSpec) bool { - return w != nil && w.Static != nil && - int32Is(w.Static.ISL, 1024) && int32Is(w.Static.OSL, 128) && int32Is(w.Static.Concurrency, 10) +// An envelope that admission has nothing to add to keeps its exact bytes, even when they are +// not in canonical form. +func TestDefaultEnvelopeV1beta2LeavesUnchangedEnvelopeUntouched(t *testing.T) { + spec := modelOnlySpecV1beta2() + defaultDynamoGraphDeploymentRequestSpecV1beta2(spec, dynamoGraphDeploymentRequestV1beta2DefaultOptions{}) + raw, err := json.Marshal(spec) + if err != nil { + t.Fatal(err) + } + dgdr := v1beta2Envelope(t, spec) + dgdr.Spec.V1Beta2.Raw = append([]byte(" "), raw...) + original := string(dgdr.Spec.V1Beta2.Raw) + if err := defaultDynamoGraphDeploymentRequestV1beta2( + context.Background(), dgdr, admissionv1.Create, nil, DGDRV1beta2Options{}); err != nil { + t.Fatal(err) + } + if string(dgdr.Spec.V1Beta2.Raw) != original { + t.Fatalf("envelope was rewritten without a change: %q", dgdr.Spec.V1Beta2.Raw) + } } - -func int32Is(p *int32, want int32) bool { return p != nil && *p == want } diff --git a/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_validation_envtest_test.go b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_validation_envtest_test.go index 2be3a73007b6..8a5b1c31796e 100644 --- a/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_validation_envtest_test.go +++ b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_validation_envtest_test.go @@ -554,7 +554,7 @@ func TestDynamoGraphDeploymentRequestValidator_Validate(t *testing.T) { gpuDiscovery: true, discovered: discoveredH200, check: func(t *testing.T, result *unstructured.Unstructured) { - expectV1beta2SchemaDefaults(t, result) + expectV1beta2StructuralDefaults(t, result) expectV1beta2AdmissionDefaults(t, result) }, }, @@ -754,6 +754,47 @@ func TestDynamoGraphDeploymentRequestValidator_Validate(t *testing.T) { "spec.aisimulate.versionOverride: Required value: is required when spec.aisimulate.image has no parseable version tag", }, }, + { + name: "v1beta2 empty objective and recommendation receive their defaults", + request: v1beta2DGDRForAdmission(func(spec map[string]any) { + spec["objective"] = map[string]any{} + spec["recommendation"] = map[string]any{} + }, nil), + gpuDiscovery: true, + discovered: discoveredH200, + check: func(t *testing.T, result *unstructured.Unstructured) { + expectV1beta2StructuralDefaults(t, result) + expectV1beta2AdmissionDefaults(t, result) + }, + }, + { + name: "v1beta2 pareto objective receives no scalar metric", + request: v1beta2DGDRForAdmission(func(spec map[string]any) { + spec["objective"] = map[string]any{"mode": "Pareto", "metrics": []any{"TTFT", "Throughput"}} + }, nil), + gpuDiscovery: true, + discovered: discoveredH200, + check: func(t *testing.T, result *unstructured.Unstructured) { + v1beta2Absent(t, result, "spec", "objective", "metric") + }, + }, + { + name: "v1beta2 explicit values are all kept", + request: v1beta2DGDRForAdmission(func(spec map[string]any) { + spec["backends"] = []any{"vllm"} + spec["hardware"] = map[string]any{"gpu": map[string]any{"sku": "h100_sxm", "budget": int64(4)}} + spec["workload"] = map[string]any{"static": map[string]any{"isl": int64(10)}} + spec["objective"] = map[string]any{"mode": "Optimize", "metric": "TTFT"} + spec["aisimulate"] = map[string]any{ + "image": "registry.example.com/aisimulate:custom", + "versionOverride": "1.4.0", + } + spec["recommendation"] = map[string]any{"maxCandidates": int64(3)} + }, nil), + gpuDiscovery: true, + discovered: discoveredH200, + check: expectV1beta2ExplicitValues, + }, { name: "v1beta2 duplicate backends are rejected on create", request: v1beta2DGDRForAdmission(func(spec map[string]any) { @@ -832,7 +873,7 @@ func TestDynamoGraphDeploymentRequestValidator_Validate(t *testing.T) { if result.GetGeneration() != 1 { t.Fatalf("generation = %d, want 1: defaulting must not change an unchanged spec", result.GetGeneration()) } - expectV1beta2SchemaDefaults(t, result) + expectV1beta2StructuralDefaults(t, result) expectV1beta2AdmissionDefaults(t, result) }, }, @@ -1235,8 +1276,8 @@ func v1beta2Empty(t *testing.T, obj *unstructured.Unstructured, path ...string) } } -// expectV1beta2SchemaDefaults checks what the CRD supplies when workload and sla are omitted. -func expectV1beta2SchemaDefaults(t *testing.T, obj *unstructured.Unstructured) { +// expectV1beta2StructuralDefaults checks the defaults admission supplies to a model-only Request. +func expectV1beta2StructuralDefaults(t *testing.T, obj *unstructured.Unstructured) { t.Helper() for key, want := range map[string]float64{"isl": 1024, "osl": 128, "concurrency": 10} { if got := v1beta2Number(t, obj, "spec", "workload", "static", key); got != want { @@ -1248,6 +1289,49 @@ func expectV1beta2SchemaDefaults(t *testing.T, obj *unstructured.Unstructured) { if got := v1beta2String(t, obj, "spec", "objective", "mode"); got != "Optimize" { t.Fatalf("objective.mode = %q, want Optimize", got) } + if got, _ := v1beta2Field(t, obj, "spec", "backends"); !reflect.DeepEqual(got, []any{"vllm", "sglang", "trtllm"}) { + t.Fatalf("backends = %#v, want vllm, sglang and trtllm", got) + } + if got := v1beta2Number(t, obj, "spec", "recommendation", "maxCandidates"); got != 1 { + t.Fatalf("recommendation.maxCandidates = %v, want 1", got) + } +} + +// expectV1beta2ExplicitValues checks that every value the explicit-values row sent survived. +func expectV1beta2ExplicitValues(t *testing.T, obj *unstructured.Unstructured) { + t.Helper() + if got, _ := v1beta2Field(t, obj, "spec", "backends"); !reflect.DeepEqual(got, []any{"vllm"}) { + t.Fatalf("backends = %#v, want only vllm", got) + } + for _, want := range []struct { + path []string + value float64 + }{ + {[]string{"spec", "hardware", "gpu", "budget"}, 4}, + {[]string{"spec", "workload", "static", "isl"}, 10}, + {[]string{"spec", "recommendation", "maxCandidates"}, 3}, + } { + if got := v1beta2Number(t, obj, want.path...); got != want.value { + t.Fatalf("%v = %v, want %v", want.path, got, want.value) + } + } + for _, want := range []struct { + path []string + value string + }{ + {[]string{"spec", "hardware", "gpu", "sku"}, "h100_sxm"}, + {[]string{"spec", "objective", "mode"}, "Optimize"}, + {[]string{"spec", "objective", "metric"}, "TTFT"}, + {[]string{"spec", "aisimulate", "image"}, "registry.example.com/aisimulate:custom"}, + {[]string{"spec", "aisimulate", "versionOverride"}, "1.4.0"}, + } { + if got := v1beta2String(t, obj, want.path...); got != want.value { + t.Fatalf("%v = %q, want %q", want.path, got, want.value) + } + } + // A static workload that names only isl is not filled in. + v1beta2Absent(t, obj, "spec", "workload", "static", "osl") + v1beta2Absent(t, obj, "spec", "workload", "static", "concurrency") } // expectV1beta2AdmissionDefaults checks what admission supplies from the environment. From cb032089437beb26d38277b02ed62d25a5041833 Mon Sep 17 00:00:00 2001 From: devivasudevan <49675305+devivasudevan@users.noreply.github.com> Date: Fri, 23 Oct 2026 12:05:28 +0200 Subject: [PATCH 36/40] fix(operator): require the admission request at the DGDR webhook boundary Resolve the admission request once in Validate and ValidateUpdate and fail when it is absent. Compare the requested version with the spec.v1beta2 carrier directly and drop RequestUsesV1Beta2, RequestIsNative and requestIsNative. The mutating handler returns the missing-request error instead of skipping defaulting. Signed-off-by: devivasudevan <49675305+devivasudevan@users.noreply.github.com> --- .../dynamographdeploymentrequest_handler.go | 8 +-- .../internal/webhook/request_version.go | 19 +------ .../internal/webhook/request_version_test.go | 23 -------- .../dynamographdeploymentrequest.go | 56 ++++++++++++++----- ...amographdeploymentrequest_boundary_test.go | 39 +++++++++++++ .../dynamographdeploymentrequest_routing.go | 29 +++++----- 6 files changed, 101 insertions(+), 73 deletions(-) create mode 100644 deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_boundary_test.go diff --git a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_handler.go b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_handler.go index 03e6474f940d..f88a48fb6c46 100644 --- a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_handler.go +++ b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_handler.go @@ -98,14 +98,14 @@ func (d *DGDRDefaulter) Default(ctx context.Context, obj runtime.Object) error { req, err := admission.RequestFromContext(ctx) if err != nil { - logger.Error(err, "failed to get admission request from context, skipping defaulting") - return nil + return fmt.Errorf("get admission request: %w", err) } + requestVersion := internalwebhook.RequestedVersion(req) // The original request version decides what spec.v1beta2 means. A native v1beta2 Request // is defaulted from its carrier on CREATE and UPDATE; only CREATE // and an UPDATE that changes spec.hardware read the cluster. - if internalwebhook.RequestUsesV1Beta2(req) && dgdr.Spec.V1Beta2 != nil { + if requestVersion == internalwebhook.VersionV1Beta2 && dgdr.Spec.V1Beta2 != nil { oldSpec, err := decodeOldSpecV1beta2(req.Operation, req.OldObject.Raw) if err != nil { return err @@ -114,7 +114,7 @@ func (d *DGDRDefaulter) Default(ctx context.Context, obj runtime.Object) error { } // A foreign-version view (a carrier written or updated through the wrong API version, or // a v1beta1 payload written through v1beta2) is read-only; validation enforces that. - if !internalwebhook.RequestIsNative(req, dgdr.Spec.V1Beta2 != nil) { + if requestVersion == internalwebhook.VersionV1Beta2 || dgdr.Spec.V1Beta2 != nil { return nil } diff --git a/deploy/operator/internal/webhook/request_version.go b/deploy/operator/internal/webhook/request_version.go index 6bc43e1a5d5f..982ecc9c3558 100644 --- a/deploy/operator/internal/webhook/request_version.go +++ b/deploy/operator/internal/webhook/request_version.go @@ -19,7 +19,9 @@ package webhook import "sigs.k8s.io/controller-runtime/pkg/webhook/admission" -const nativeCarrierVersion = "v1beta2" +// VersionV1Beta2 is the API version whose Requests reach the webhooks as a converted v1beta1 +// object carrying spec.v1beta2. +const VersionV1Beta2 = "v1beta2" // RequestedVersion is the API version the client used. With matchPolicy: Equivalent the API // server converts the object to the version the webhook registered for; RequestKind then @@ -31,18 +33,3 @@ func RequestedVersion(req admission.Request) string { } return req.Kind.Version } - -// RequestUsesV1Beta2 reports whether the client addressed the Request through v1beta2. -func RequestUsesV1Beta2(req admission.Request) bool { - return RequestedVersion(req) == nativeCarrierVersion -} - -// RequestIsNative reports whether the represented spec is native to the API version the -// client used. hasV1Beta2Carrier is whether the converted v1beta1 object carries spec.v1beta2. -// After conversion that carrier means the represented object is native v1beta2, so the -// request is native exactly when both agree. A mismatch is a foreign-version view: the -// client either wrote a conversion carrier directly or reads or updates an object through -// the API version that does not own it. -func RequestIsNative(req admission.Request, hasV1Beta2Carrier bool) bool { - return RequestUsesV1Beta2(req) == hasV1Beta2Carrier -} diff --git a/deploy/operator/internal/webhook/request_version_test.go b/deploy/operator/internal/webhook/request_version_test.go index 3a6680142eda..837bf346d0a4 100644 --- a/deploy/operator/internal/webhook/request_version_test.go +++ b/deploy/operator/internal/webhook/request_version_test.go @@ -55,26 +55,3 @@ func TestRequestedVersion(t *testing.T) { }) } } - -func TestRequestIsNative(t *testing.T) { - tests := []struct { - name string - req admission.Request - carrier bool - want bool - }{ - {"native v1beta2 create", requestWithKinds("v1beta2"), true, true}, - {"native v1beta1 create", requestWithKinds(""), false, true}, - {"native v1alpha1 create", requestWithKinds("v1alpha1"), false, true}, - {"carrier written through v1beta1", requestWithKinds(""), true, false}, - {"carrier written through v1alpha1", requestWithKinds("v1alpha1"), true, false}, - {"v1beta1 payload written through v1beta2", requestWithKinds("v1beta2"), false, false}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := RequestIsNative(tt.req, tt.carrier); got != tt.want { - t.Fatalf("RequestIsNative() = %v, want %v", got, tt.want) - } - }) - } -} diff --git a/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest.go b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest.go index 3315b26d7bf7..85e6b28a1695 100644 --- a/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest.go +++ b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest.go @@ -28,6 +28,7 @@ import ( "github.com/ai-dynamo/dynamo/deploy/operator/internal/dgdoverride" "github.com/ai-dynamo/dynamo/deploy/operator/internal/features" "github.com/ai-dynamo/dynamo/deploy/operator/internal/versionoverride" + internalwebhook "github.com/ai-dynamo/dynamo/deploy/operator/internal/webhook" apiequality "k8s.io/apimachinery/pkg/api/equality" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" k8sruntime "k8s.io/apimachinery/pkg/runtime" @@ -46,31 +47,51 @@ func NewDynamoGraphDeploymentRequestValidator() *DynamoGraphDeploymentRequestVal // dynamoGraphDeploymentRequestValidation carries DGDR-specific request state. // API values, paths, and accumulated errors remain explicit validator arguments. type dynamoGraphDeploymentRequestValidation struct { - ctx context.Context - warnings admission.Warnings + ctx context.Context + // requestVersion is the API version the client used, resolved once from the admission + // request. Every object delivered to the webhook is a converted v1beta1 object; + // spec.v1beta2 says which represented spec it carries. + requestVersion string + warnings admission.Warnings } func (v *dynamoGraphDeploymentRequestValidation) warn(message string) { v.warnings = append(v.warnings, message) } -// Validate performs stateless validation on request. ctx and request must not be nil. +// Validate performs stateless validation on request. ctx must carry the admission request and +// request must not be nil. func (v *DynamoGraphDeploymentRequestValidator) Validate( ctx context.Context, request *nvidiacomv1beta1.DynamoGraphDeploymentRequest, ) (admission.Warnings, error) { - validation := &dynamoGraphDeploymentRequestValidation{ctx: ctx} + req, err := admission.RequestFromContext(ctx) + if err != nil { + return nil, fmt.Errorf("get admission request: %w", err) + } + validation := &dynamoGraphDeploymentRequestValidation{ + ctx: ctx, + requestVersion: internalwebhook.RequestedVersion(req), + } allErrs := validation.validateDynamoGraphDeploymentRequest(request) return validation.warnings, invalidDynamoGraphDeploymentRequestError(request, allErrs) } -// ValidateUpdate validates newRequest against oldRequest. ctx, oldRequest, and newRequest must not be nil. +// ValidateUpdate validates newRequest against oldRequest. ctx must carry the admission request; +// oldRequest and newRequest must not be nil. func (v *DynamoGraphDeploymentRequestValidator) ValidateUpdate( ctx context.Context, oldRequest *nvidiacomv1beta1.DynamoGraphDeploymentRequest, newRequest *nvidiacomv1beta1.DynamoGraphDeploymentRequest, ) (admission.Warnings, error) { - validation := &dynamoGraphDeploymentRequestValidation{ctx: ctx} + req, err := admission.RequestFromContext(ctx) + if err != nil { + return nil, fmt.Errorf("get admission request: %w", err) + } + validation := &dynamoGraphDeploymentRequestValidation{ + ctx: ctx, + requestVersion: internalwebhook.RequestedVersion(req), + } allErrs := validation.validateDynamoGraphDeploymentRequestUpdate(newRequest, oldRequest) validation.warnDeprecatedDGDOverrideTargets(&newRequest.Spec) validation.warnDeprecatedGPUSKU(&newRequest.Spec) @@ -89,10 +110,15 @@ func (v *dynamoGraphDeploymentRequestValidation) validateDynamoGraphDeploymentRe spec *nvidiacomv1beta1.DynamoGraphDeploymentRequestSpec, fldPath *field.Path, ) field.ErrorList { - // The requested API version decides what spec.v1beta2 means. A native request is - // validated; a spec that only conversion may produce is rejected on CREATE. - if !v.requestIsNative(spec.V1Beta2 != nil) { - return foreignCreateErrors(fldPath, spec.V1Beta2 != nil) + // The requested API version decides what spec.v1beta2 means. A spec that only conversion + // may produce is rejected on CREATE. + if v.requestVersion == internalwebhook.VersionV1Beta2 && spec.V1Beta2 == nil { + // A hub/v1beta1 representation was submitted through v1beta2. + return foreignCreateErrors(fldPath, false) + } + if v.requestVersion != internalwebhook.VersionV1Beta2 && spec.V1Beta2 != nil { + // A v1beta2 carrier was submitted through v1alpha1 or v1beta1. + return foreignCreateErrors(fldPath, true) } // A v1beta2 Request reaches this webhook as a v1beta1 object carrying spec.v1beta2; // the v1beta1 rules below do not apply to it. @@ -254,11 +280,11 @@ func (v *dynamoGraphDeploymentRequestValidation) validateDynamoGraphDeploymentRe ) field.ErrorList { // A spec viewed through the API version that does not own it is read-only: an unchanged // spec round-trips, any other write is rejected. - if !v.requestIsNative(newSpec.V1Beta2 != nil) { - if apiequality.Semantic.DeepEqual(newSpec, oldSpec) { - return nil - } - return foreignUpdateErrors(fldPath, newSpec.V1Beta2 != nil) + if v.requestVersion == internalwebhook.VersionV1Beta2 && newSpec.V1Beta2 == nil { + return foreignUpdateVerdict(fldPath, newSpec, oldSpec) + } + if v.requestVersion != internalwebhook.VersionV1Beta2 && newSpec.V1Beta2 != nil { + return foreignUpdateVerdict(fldPath, newSpec, oldSpec) } // A native v1beta2 spec is unconstrained by phase. Stateless validation of the new typed // spec runs on every CREATE and UPDATE: an unchanged spec is no reason to skip it. Ratcheting diff --git a/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_boundary_test.go b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_boundary_test.go new file mode 100644 index 000000000000..e8fdc85c88c2 --- /dev/null +++ b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_boundary_test.go @@ -0,0 +1,39 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package validation + +import ( + "context" + "testing" + + nvidiacomv1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" +) + +// The webhook boundary protects raw conversion carriers, so a missing admission request must +// fail the call instead of counting as a native write. +func TestDGDRValidatorRequiresTheAdmissionRequest(t *testing.T) { + validator := NewDynamoGraphDeploymentRequestValidator() + request := &nvidiacomv1beta1.DynamoGraphDeploymentRequest{} + + if _, err := validator.Validate(context.Background(), request); err == nil { + t.Fatal("Validate must fail without an admission request in the context") + } + if _, err := validator.ValidateUpdate(context.Background(), request, request); err == nil { + t.Fatal("ValidateUpdate must fail without an admission request in the context") + } +} diff --git a/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_routing.go b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_routing.go index 71a6419cdd8c..a0c10bc6fc31 100644 --- a/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_routing.go +++ b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_routing.go @@ -18,24 +18,11 @@ package validation import ( - internalwebhook "github.com/ai-dynamo/dynamo/deploy/operator/internal/webhook" + nvidiacomv1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" + apiequality "k8s.io/apimachinery/pkg/api/equality" "k8s.io/apimachinery/pkg/util/validation/field" - "sigs.k8s.io/controller-runtime/pkg/webhook/admission" ) -// requestIsNative reports whether the represented spec is native to the API version the -// client used (see internalwebhook.RequestIsNative). hasV1Beta2Carrier is whether the -// converted v1beta1 object carries spec.v1beta2. Without an admission request in the -// context (direct calls from unit tests) there is no foreign view, so the request counts as -// native. -func (v *dynamoGraphDeploymentRequestValidation) requestIsNative(hasV1Beta2Carrier bool) bool { - req, err := admission.RequestFromContext(v.ctx) - if err != nil { - return true - } - return internalwebhook.RequestIsNative(req, hasV1Beta2Carrier) -} - // foreignCreateErrors rejects a spec that conversion alone may produce. The carrier is raw // JSON that bypasses structural pruning; it is safe only when conversion built it from a // native request that was validated against its own versioned schema. @@ -52,6 +39,18 @@ func foreignCreateErrors(fldPath *field.Path, hasV1Beta2Carrier bool) field.Erro )} } +// foreignUpdateVerdict decides an UPDATE whose spec is viewed through the API version that +// does not own it: an unchanged spec round-trips, any other write is rejected. +func foreignUpdateVerdict( + fldPath *field.Path, + newSpec, oldSpec *nvidiacomv1beta1.DynamoGraphDeploymentRequestSpec, +) field.ErrorList { + if apiequality.Semantic.DeepEqual(newSpec, oldSpec) { + return nil + } + return foreignUpdateErrors(fldPath, newSpec.V1Beta2 != nil) +} + // foreignUpdateErrors rejects a change to a spec viewed through the API version that does // not own it. An unchanged spec never reaches this function. func foreignUpdateErrors(fldPath *field.Path, hasV1Beta2Carrier bool) field.ErrorList { From 64da2a120ff3c248c2e9cdf516d179b3778290f6 Mon Sep 17 00:00:00 2001 From: devivasudevan <49675305+devivasudevan@users.noreply.github.com> Date: Fri, 23 Oct 2026 12:30:50 +0200 Subject: [PATCH 37/40] fix(operator): ask aisimulate whether the input names hardware The admission change made the AISimulate path variables private. The Request controller read two of them to check for a GPU SKU and budget. Expose that answer as aisimulate.HardwareNamed instead. Signed-off-by: devivasudevan <49675305+devivasudevan@users.noreply.github.com> --- .../aisimulate/hardware_named_test.go | 43 +++++++++++++++++++ .../internal/controller/aisimulate/paths.go | 16 +++++++ ...aphdeploymentrequest_v1beta2_controller.go | 14 +----- 3 files changed, 60 insertions(+), 13 deletions(-) create mode 100644 deploy/operator/internal/controller/aisimulate/hardware_named_test.go diff --git a/deploy/operator/internal/controller/aisimulate/hardware_named_test.go b/deploy/operator/internal/controller/aisimulate/hardware_named_test.go new file mode 100644 index 000000000000..fdc1e2167158 --- /dev/null +++ b/deploy/operator/internal/controller/aisimulate/hardware_named_test.go @@ -0,0 +1,43 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package aisimulate + +import "testing" + +func TestHardwareNamed(t *testing.T) { + tests := []struct { + name string + input map[string]any + want bool + }{ + {"both named", map[string]any{"search_space": map[string]any{"hardware_sku": "h200_sxm", "gpu_budget": int64(8)}}, true}, + {"no search space", map[string]any{}, false}, + {"budget missing", map[string]any{"search_space": map[string]any{"hardware_sku": "h200_sxm"}}, false}, + {"sku missing", map[string]any{"search_space": map[string]any{"gpu_budget": int64(8)}}, false}, + {"null sku", map[string]any{"search_space": map[string]any{"hardware_sku": nil, "gpu_budget": int64(8)}}, false}, + {"blank sku", map[string]any{"search_space": map[string]any{"hardware_sku": "", "gpu_budget": int64(8)}}, false}, + {"null budget", map[string]any{"search_space": map[string]any{"hardware_sku": "h200_sxm", "gpu_budget": nil}}, false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := HardwareNamed(tt.input); got != tt.want { + t.Fatalf("HardwareNamed() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/deploy/operator/internal/controller/aisimulate/paths.go b/deploy/operator/internal/controller/aisimulate/paths.go index bd2a9d16b7d8..8a2c13264f2a 100644 --- a/deploy/operator/internal/controller/aisimulate/paths.go +++ b/deploy/operator/internal/controller/aisimulate/paths.go @@ -57,3 +57,19 @@ func has(obj map[string]any, path ...string) bool { _, found, _ := unstructured.NestedFieldNoCopy(obj, path...) return found } + +// HardwareNamed reports whether the native input names both a GPU SKU and a GPU budget, as the +// search needs. A null or blank value does not count: the key is deleted or left empty. +func HardwareNamed(input map[string]any) bool { + return hasValue(input, searchSpaceHardwareSKU...) && hasValue(input, searchSpaceGPUBudget...) +} + +// hasValue reports whether obj holds a non-null, non-blank value at the path. +func hasValue(obj map[string]any, path ...string) bool { + value, found, err := unstructured.NestedFieldNoCopy(obj, path...) + if err != nil || !found || value == nil { + return false + } + text, isString := value.(string) + return !isString || text != "" +} diff --git a/deploy/operator/internal/controller/dynamographdeploymentrequest_v1beta2_controller.go b/deploy/operator/internal/controller/dynamographdeploymentrequest_v1beta2_controller.go index 84df14ccf887..605895009fa3 100644 --- a/deploy/operator/internal/controller/dynamographdeploymentrequest_v1beta2_controller.go +++ b/deploy/operator/internal/controller/dynamographdeploymentrequest_v1beta2_controller.go @@ -38,7 +38,6 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" ctrl "sigs.k8s.io/controller-runtime" @@ -219,8 +218,7 @@ func (r *DynamoGraphDeploymentRequestV1beta2Reconciler) buildRun( if err != nil { return nil, err } - if !dgdrV1beta2HasEffectiveValue(input, aisimulate.SearchSpaceHardwareSKU) || - !dgdrV1beta2HasEffectiveValue(input, aisimulate.SearchSpaceGPUBudget) { + if !aisimulate.HardwareNamed(input) { return nil, errors.New("the effective AISimulate input names no GPU SKU or budget; set spec.hardware.gpu") } raw, err := json.Marshal(input) @@ -402,16 +400,6 @@ func dgdrV1beta2SplitRuns( return active, terminal, nil } -// dgdrV1beta2HasEffectiveValue reports whether input holds a non-null, non-empty value at path. -func dgdrV1beta2HasEffectiveValue(input map[string]any, path []string) bool { - value, found, err := unstructured.NestedFieldNoCopy(input, path...) - if err != nil || !found || value == nil { - return false - } - text, isString := value.(string) - return !isString || text != "" -} - // requireDGDRV1beta2API fails when the cluster does not serve the v1beta2 Request and Run APIs. // Without them the controller's caches never sync and the manager would hang at startup. func requireDGDRV1beta2API(mgr ctrl.Manager) error { From f8674b9e9565c67f2bd8ac3511ef4f3c4117944d Mon Sep 17 00:00:00 2001 From: "Dr. Stefan Schimanski" Date: Fri, 23 Oct 2026 18:35:35 +0200 Subject: [PATCH 38/40] planner/dgdr: derive run candidate images Signed-off-by: Dr. Stefan Schimanski --- .../aisimulate/output/dgdr_run/adapter.py | 22 ++++-- .../aisimulate/tests/dgdr_run/test_adapter.py | 59 ++++++++++++++ .../src/dynamo/common/container_images.py | 40 ++++++++++ .../dynamo/profiler/utils/profile_common.py | 78 +------------------ 4 files changed, 118 insertions(+), 81 deletions(-) create mode 100644 components/src/dynamo/common/container_images.py diff --git a/components/src/dynamo/aisimulate/output/dgdr_run/adapter.py b/components/src/dynamo/aisimulate/output/dgdr_run/adapter.py index 112c1331ae40..72869b16cacd 100644 --- a/components/src/dynamo/aisimulate/output/dgdr_run/adapter.py +++ b/components/src/dynamo/aisimulate/output/dgdr_run/adapter.py @@ -74,6 +74,7 @@ SnapshotCandidate, write_snapshot, ) +from dynamo.common.container_images import derive_backend_image logger = logging.getLogger(__name__) @@ -177,14 +178,22 @@ class DGDRRunOutputConfig(BaseModel): namespace: str | None = None generator: Literal["aic", "direct"] = "aic" runtime_image: str = Field(min_length=1) + # A Run searches multiple backends from one planner image. Standalone callers keep + # the historical fixed-image behavior unless they opt into per-candidate derivation. + derive_runtime_images: bool = False runtime_version_override: str | None = None num_gpus_per_node: int = Field(gt=0) max_candidates: int = Field(default=5, ge=1) snapshot_interval_seconds: float = Field(default=5.0, ge=0) - def generation_options(self) -> DGDGenerationOptions: + def generation_options(self, backend: str) -> DGDGenerationOptions: + runtime_image = ( + derive_backend_image(self.runtime_image, backend) + if self.derive_runtime_images + else self.runtime_image + ) return DGDGenerationOptions( - runtime_image=self.runtime_image, + runtime_image=runtime_image, runtime_version_override=self.runtime_version_override, namespace=self.namespace, num_gpus_per_node=self.num_gpus_per_node, @@ -222,7 +231,8 @@ def __init__( raise ValueError("a snapshot directory is required") self._directory = Path(directory) self._config = config - self._options = config.generation_options() + # Validate the source image and runtime version before the first callback. + config.generation_options("vllm") self._workload = workload self._max_candidates = config.max_candidates self._interval = config.snapshot_interval_seconds @@ -504,7 +514,9 @@ def _materialize(self, item: _Retained) -> SnapshotCandidate: manifest = render_dgd( _RenderInput(config=item.parameters, used_gpus=item.used_gpus), self._workload, - self._options, + self._config.generation_options( + str(item.parameters.get("backend", "")) + ), dgd_name=f"{self._config.name}-{item.id.removeprefix(_ID_PREFIX)}", renderer=self._config.generator, ) @@ -564,7 +576,7 @@ def subscribe( snapshot is still produced by ``write()``. """ resolved = DGDRRunOutputConfig.model_validate(dict(config)) - resolved.generation_options() + resolved.generation_options("vllm") workload = getattr(context, "workload", None) if workload is None or resolved.snapshot_dir is None: return None diff --git a/components/src/dynamo/aisimulate/tests/dgdr_run/test_adapter.py b/components/src/dynamo/aisimulate/tests/dgdr_run/test_adapter.py index eb36b89b11b7..7dd3b78732ed 100644 --- a/components/src/dynamo/aisimulate/tests/dgdr_run/test_adapter.py +++ b/components/src/dynamo/aisimulate/tests/dgdr_run/test_adapter.py @@ -181,6 +181,65 @@ def test_snapshot_preserves_the_evaluated_point_not_just_its_manifest( assert "marker: a" in candidate["manifest"] +def test_each_candidate_uses_the_runtime_image_for_its_backend( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + images: dict[str, str] = {} + + def capture_image( + candidate: Any, workload: Any, options: Any, **kwargs: Any + ) -> str: + del workload, kwargs + images[candidate.config["backend"]] = options.runtime_image + return "apiVersion: nvidia.com/v1beta1\nkind: DynamoGraphDeployment\n" + + monkeypatch.setattr(adapter_module, "render_dgd", capture_image) + adapter = DGDRRunOutputAdapter( + make_config( + tmp_path, + runtime_image="nvcr.io/nvidia/ai-dynamo/dynamo-planner:1.2.3", + derive_runtime_images=True, + ), + workload=None, + ) + for index, backend in enumerate(("vllm", "sglang", "trtllm")): + candidate = record(backend, float(index)) + candidate.config["backend"] = backend + adapter.on_candidate(candidate) + adapter.close() + + assert images == { + "vllm": "nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.2.3", + "sglang": "nvcr.io/nvidia/ai-dynamo/sglang-runtime:1.2.3", + "trtllm": "nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.2.3", + } + + +def test_explicit_runtime_image_stays_fixed_by_default( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + images: list[str] = [] + + def capture_image( + candidate: Any, workload: Any, options: Any, **kwargs: Any + ) -> str: + del candidate, workload, kwargs + images.append(options.runtime_image) + return "apiVersion: nvidia.com/v1beta1\nkind: DynamoGraphDeployment\n" + + monkeypatch.setattr(adapter_module, "render_dgd", capture_image) + adapter = DGDRRunOutputAdapter( + make_config(tmp_path, runtime_image="example.com/my-runtime:1.2.3"), + workload=None, + ) + candidate = record("sglang", 1.0) + candidate.config["backend"] = "sglang" + adapter.on_candidate(candidate) + adapter.close() + + assert images == ["example.com/my-runtime:1.2.3"] + + def test_round_progress_is_monotonic(tmp_path: Path, renders: list[str]) -> None: adapter = DGDRRunOutputAdapter(make_config(tmp_path), workload=None) adapter.on_round(3, []) diff --git a/components/src/dynamo/common/container_images.py b/components/src/dynamo/common/container_images.py new file mode 100644 index 000000000000..4e00b491a297 --- /dev/null +++ b/components/src/dynamo/common/container_images.py @@ -0,0 +1,40 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Published Dynamo container-image naming conventions.""" + +BACKEND_IMAGE_NAMES: dict[str, str] = { + "vllm": "vllm-runtime", + "sglang": "sglang-runtime", + "trtllm": "tensorrtllm-runtime", +} + +PLANNER_IMAGE_NAME = "dynamo-planner" + + +def _replace_image_name(image_ref: str, new_name: str) -> str: + """Replace the image name while preserving its registry, tag and digest.""" + slash_idx = image_ref.rfind("/") + prefix = image_ref[: slash_idx + 1] if slash_idx >= 0 else "" + suffix = image_ref[slash_idx + 1 :] + name_and_tag, has_digest, digest = suffix.partition("@") + colon_idx = name_and_tag.rfind(":") + tag = name_and_tag[colon_idx:] if colon_idx >= 0 else "" + digest_suffix = f"@{digest}" if has_digest else "" + return f"{prefix}{new_name}{tag}{digest_suffix}" + + +def derive_planner_image(source_image: str) -> str: + """Derive the planner image that belongs to a published Dynamo image.""" + return _replace_image_name(source_image, PLANNER_IMAGE_NAME) + + +def derive_backend_image(source_image: str, backend: str) -> str: + """Derive one backend runtime image from a published Dynamo image.""" + backend_image_name = BACKEND_IMAGE_NAMES.get(backend) + if backend_image_name is None: + raise ValueError( + f"Cannot derive backend image for unknown backend '{backend}'. " + f"Supported backends: {list(BACKEND_IMAGE_NAMES.keys())}" + ) + return _replace_image_name(source_image, backend_image_name) diff --git a/components/src/dynamo/profiler/utils/profile_common.py b/components/src/dynamo/profiler/utils/profile_common.py index 4fc42e335c8d..c01b06c312fc 100644 --- a/components/src/dynamo/profiler/utils/profile_common.py +++ b/components/src/dynamo/profiler/utils/profile_common.py @@ -23,6 +23,8 @@ import pandas as pd +from dynamo.common.container_images import derive_backend_image as derive_backend_image +from dynamo.common.container_images import derive_planner_image as derive_planner_image from dynamo.planner.config.planner_config import PlannerPreDeploymentSweepMode from dynamo.profiler.utils.config_modifiers.parallelization_mapping import ( PickedParallelConfig, @@ -36,82 +38,6 @@ logger = logging.getLogger(__name__) -# --------------------------------------------------------------------------- -# Published container image naming conventions -# --------------------------------------------------------------------------- - -# Mapping from backend name to the image-name component of the published -# backend runtime image. -# e.g. vllm → nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.1.1 -BACKEND_IMAGE_NAMES: dict[str, str] = { - "vllm": "vllm-runtime", - "sglang": "sglang-runtime", - "trtllm": "tensorrtllm-runtime", -} - -PLANNER_IMAGE_NAME = "dynamo-planner" - - -def _replace_image_name(image_ref: str, new_name: str) -> str: - """Replace the image name component in a Docker image reference. - - Preserves the registry path prefix and tag suffix, only replacing the - last ``/``-delimited component (before any ``:tag``). - """ - slash_idx = image_ref.rfind("/") - prefix = image_ref[: slash_idx + 1] if slash_idx >= 0 else "" - suffix = image_ref[slash_idx + 1 :] - name_and_tag, has_digest, digest = suffix.partition("@") - colon_idx = name_and_tag.rfind(":") - tag = name_and_tag[colon_idx:] if colon_idx >= 0 else "" - digest_suffix = f"@{digest}" if has_digest else "" - return f"{prefix}{new_name}{tag}{digest_suffix}" - - -def derive_planner_image(profiler_image: str) -> str: - """Derive the planner service image from the profiler image reference.""" - return _replace_image_name(profiler_image, PLANNER_IMAGE_NAME) - - -def derive_backend_image(profiler_image: str, backend: str) -> str: - """Derive the backend worker image from the profiler image. - - Replaces the image name (the last ``/``-delimited component, before any - ``:tag``) with the backend-specific runtime image name, preserving the - registry path and tag unchanged. - - Examples:: - - derive_backend_image( - "nvcr.io/nvidia/ai-dynamo/dynamo-planner:1.1.1", "vllm" - ) - # → "nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.1.1" - - derive_backend_image("myregistry.io/sglang-runtime:1.1.1", "sglang") - # → "myregistry.io/sglang-runtime:1.1.1" - - Args: - profiler_image: Any Docker image reference of the form - ``[REGISTRY/]NAME[:TAG]``. - backend: The resolved backend type (``'vllm'``, ``'sglang'``, or - ``'trtllm'``). - - Returns: - The backend container image string. - - Raises: - ValueError: If *backend* is not a recognised backend. - """ - backend_image_name = BACKEND_IMAGE_NAMES.get(backend) - if backend_image_name is None: - raise ValueError( - f"Cannot derive backend image for unknown backend '{backend}'. " - f"Supported backends: {list(BACKEND_IMAGE_NAMES.keys())}" - ) - - return _replace_image_name(profiler_image, backend_image_name) - - # --------------------------------------------------------------------------- # Operational defaults not part of DynamoGraphDeploymentRequestSpec # --------------------------------------------------------------------------- From 2d18e5dd03b4668500f63b998a963288c899da9d Mon Sep 17 00:00:00 2001 From: "Dr. Stefan Schimanski" Date: Fri, 23 Oct 2026 18:36:58 +0200 Subject: [PATCH 39/40] operator/dgdr: execute v1beta2 runs Signed-off-by: Dr. Stefan Schimanski --- .../components/operator/files/role.yaml | 22 +- deploy/operator/Dockerfile | 4 +- deploy/operator/cmd/main.go | 7 + .../internal/controller/aisimulate/input.go | 28 +- .../internal/controller/aisimulate/run.go | 55 -- .../controller/aisimulate/run_test.go | 58 -- ...aphdeploymentrequest_v1beta2_controller.go | 11 +- ...request_v1beta2_controller_envtest_test.go | 2 +- ...ploymentrequest_v1beta2_controller_test.go | 4 +- ...mographdeploymentrun_v1beta2_controller.go | 665 ++++++++++++++++++ ...phdeploymentrun_v1beta2_controller_test.go | 392 +++++++++++ deploy/operator/internal/controller/setup.go | 46 ++ 12 files changed, 1167 insertions(+), 127 deletions(-) delete mode 100644 deploy/operator/internal/controller/aisimulate/run.go delete mode 100644 deploy/operator/internal/controller/aisimulate/run_test.go create mode 100644 deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller.go create mode 100644 deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller_test.go diff --git a/deploy/helm/charts/platform/components/operator/files/role.yaml b/deploy/helm/charts/platform/components/operator/files/role.yaml index bc0bcfaa36b8..c33fb4afee55 100644 --- a/deploy/helm/charts/platform/components/operator/files/role.yaml +++ b/deploy/helm/charts/platform/components/operator/files/role.yaml @@ -24,9 +24,10 @@ rules: - "" resources: - nodes - - pods/log verbs: - get + - list + - watch - apiGroups: - "" resources: @@ -48,15 +49,31 @@ rules: - list - patch - watch +- apiGroups: + - "" + resources: + - pods/log + verbs: + - get - apiGroups: - "" resources: - secrets + verbs: + - create + - get + - list + - update + - watch +- apiGroups: + - "" + resources: - serviceaccounts verbs: - create - get - list + - patch - update - watch - apiGroups: @@ -296,6 +313,7 @@ rules: resources: - dynamocomponentdeployments/status - dynamographdeploymentrequests/status + - dynamographdeploymentruns/status - dynamographdeployments/status - dynamographdeploymentscalingadapters/status - dynamomodels/status @@ -352,6 +370,7 @@ rules: - delete - get - list + - patch - update - watch - apiGroups: @@ -362,6 +381,7 @@ rules: - create - get - list + - patch - update - watch - apiGroups: diff --git a/deploy/operator/Dockerfile b/deploy/operator/Dockerfile index 35f9835e5aa6..09886f121aea 100644 --- a/deploy/operator/Dockerfile +++ b/deploy/operator/Dockerfile @@ -53,6 +53,7 @@ FROM base AS builder RUN CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} go build -o manager ./cmd/main.go && \ CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} go build -o crd-apply ./cmd/crd-apply/main.go && \ + CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} go build -trimpath -ldflags="-s -w" -o dgdr-run-publisher ./cmd/dgdr-run-publisher && \ CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} go build -trimpath -ldflags="-s -w" -o dgd-apply-overrides ./cmd/dgd-apply-overrides # Emit a CycloneDX SBOM for the operator's Go module set so the licenses @@ -60,7 +61,7 @@ RUN CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} go build -o manager ./cm # v1.7.0 to keep the SBOM tool itself reproducible. # # `app -main ./cmd` covers the manager binary (./cmd/main.go = package -# main in ./cmd). crd-apply and dgd-apply-overrides are separate mains in +# main in ./cmd). crd-apply, dgdr-run-publisher and dgd-apply-overrides are separate mains in # the same module, so their deps are a subset of manager's — a single SBOM # is sufficient. Tried `mod` first but its license detection is weaker than # `app`'s — everything came back UNKNOWN. @@ -208,6 +209,7 @@ FROM nvcr.io/nvidia/distroless/go:v4.1.5 WORKDIR / COPY --from=builder /workspace/manager . COPY --from=builder /workspace/crd-apply . +COPY --from=builder /workspace/dgdr-run-publisher . COPY --from=builder /workspace/dgd-apply-overrides . COPY --from=builder /workspace/config/crd/bases/ /opt/dynamo-operator/crds/ COPY --from=licenses /legal /legal diff --git a/deploy/operator/cmd/main.go b/deploy/operator/cmd/main.go index 98c73b32d4e7..a3ab590bd786 100644 --- a/deploy/operator/cmd/main.go +++ b/deploy/operator/cmd/main.go @@ -626,6 +626,13 @@ func registerControllers( if err := controller.SetupDynamoGraphDeploymentRequestV1beta2(mgr, setupOptions); err != nil { return err } + if err := controller.SetupDynamoGraphDeploymentRunV1beta2(mgr, controller.DynamoGraphDeploymentRunSetupOptions{ + SetupOptions: setupOptions, + OperatorImage: operatorImage, + OperatorImagePullPolicy: operatorPullPolicy, + }); err != nil { + return err + } } if err := controller.SetupDynamoModel(mgr, controller.DynamoModelSetupOptions{ SetupOptions: setupOptions, diff --git a/deploy/operator/internal/controller/aisimulate/input.go b/deploy/operator/internal/controller/aisimulate/input.go index fbfee0514080..4eaae415c8ad 100644 --- a/deploy/operator/internal/controller/aisimulate/input.go +++ b/deploy/operator/internal/controller/aisimulate/input.go @@ -25,6 +25,7 @@ import ( v1beta2 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta2" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" ) // Lower returns the effective native input: the typed underlay, merged with @@ -46,11 +47,18 @@ func Lower(spec *v1beta2.DynamoGraphDeploymentRequestSpec) (map[string]any, erro if err := applyDefaults(underlay); err != nil { return nil, err } + return MergeParameters(underlay, parameters), nil +} + +// MergeParameters applies parameters as an RFC 7386 merge patch over underlay. Native +// AISimulate parameters are always the final overlay: they may override or delete values +// supplied by a typed API or by controller-owned execution defaults. +func MergeParameters(underlay, parameters map[string]any) map[string]any { input, _ := mergePatch(underlay, parameters).(map[string]any) if input == nil { - input = map[string]any{} + return map[string]any{} } - return input, nil + return input } // Parameters returns spec.aisimulate.parameters as a JSON object, or nil when absent. @@ -58,10 +66,20 @@ func Parameters(spec *v1beta2.DynamoGraphDeploymentRequestSpec) (map[string]any, if spec.AISimulate == nil || spec.AISimulate.Parameters == nil { return nil, nil } - raw := spec.AISimulate.Parameters.Raw - if len(raw) == 0 && spec.AISimulate.Parameters.Object != nil { + return DecodeParameters(spec.AISimulate.Parameters) +} + +// DecodeParameters returns a native AISimulate parameters object without applying typed +// API fields or defaults. A Run controller uses this after the Request controller has frozen +// the complete effective input into spec.aisimulate.parameters. +func DecodeParameters(parameters *runtime.RawExtension) (map[string]any, error) { + if parameters == nil { + return nil, nil + } + raw := parameters.Raw + if len(raw) == 0 && parameters.Object != nil { var err error - if raw, err = json.Marshal(spec.AISimulate.Parameters.Object); err != nil { + if raw, err = json.Marshal(parameters.Object); err != nil { return nil, fmt.Errorf("spec.aisimulate.parameters: %w", err) } } diff --git a/deploy/operator/internal/controller/aisimulate/run.go b/deploy/operator/internal/controller/aisimulate/run.go deleted file mode 100644 index 0c6ce9a1f631..000000000000 --- a/deploy/operator/internal/controller/aisimulate/run.go +++ /dev/null @@ -1,55 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package aisimulate - -import ( - "encoding/json" - "errors" - "fmt" - - "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta2" -) - -// InputForRun returns the AISimulate input a Run executes. run.spec.aisimulate.parameters is the -// complete resolved native input, frozen when the Run was created. Run execution must use this -// object directly and must not pass the Run through Request lowering (Lower): the typed fields of -// the Run are copies of the Request and would bring back keys that spec.aisimulate.parameters -// deleted with a JSON merge patch null, and the nulls themselves are no longer in the stored result. -func InputForRun(run *v1beta2.DynamoGraphDeploymentRun) (map[string]any, error) { - if run.Spec.AISimulate == nil || run.Spec.AISimulate.Parameters == nil { - return nil, errors.New("run has no spec.aisimulate.parameters") - } - raw := run.Spec.AISimulate.Parameters.Raw - if len(raw) == 0 && run.Spec.AISimulate.Parameters.Object != nil { - var err error - if raw, err = json.Marshal(run.Spec.AISimulate.Parameters.Object); err != nil { - return nil, fmt.Errorf("run spec.aisimulate.parameters: %w", err) - } - } - if len(raw) == 0 { - return nil, errors.New("run spec.aisimulate.parameters is empty") - } - var input map[string]any - if err := json.Unmarshal(raw, &input); err != nil { - return nil, fmt.Errorf("run spec.aisimulate.parameters must be a JSON object: %w", err) - } - if len(input) == 0 { - return nil, errors.New("run spec.aisimulate.parameters is empty") - } - return input, nil -} diff --git a/deploy/operator/internal/controller/aisimulate/run_test.go b/deploy/operator/internal/controller/aisimulate/run_test.go deleted file mode 100644 index 1373c8c9536c..000000000000 --- a/deploy/operator/internal/controller/aisimulate/run_test.go +++ /dev/null @@ -1,58 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package aisimulate - -import ( - "testing" - - "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta2" - "k8s.io/apimachinery/pkg/runtime" -) - -func runWithParameters(raw string) *v1beta2.DynamoGraphDeploymentRun { - run := &v1beta2.DynamoGraphDeploymentRun{} - run.Spec.AISimulate = &v1beta2.AISimulateSpec{Parameters: &runtime.RawExtension{Raw: []byte(raw)}} - return run -} - -func TestInputForRunReturnsTheFrozenObjectUnchanged(t *testing.T) { - input, err := InputForRun(runWithParameters(`{"search_space":{"gpu_budget":4},"workload":{"osl":128}}`)) - if err != nil { - t.Fatalf("InputForRun: %v", err) - } - workload, _ := input["workload"].(map[string]any) - if _, found := workload["isl"]; found || workload["osl"] != float64(128) { - t.Fatalf("workload = %v, want exactly the stored keys", workload) - } -} - -func TestInputForRunRejectsAMissingOrMalformedInput(t *testing.T) { - for name, run := range map[string]*v1beta2.DynamoGraphDeploymentRun{ - "no aisimulate": {}, - "no parameters": {Spec: v1beta2.DynamoGraphDeploymentRunSpec{ - NativeSearchInput: v1beta2.NativeSearchInput{AISimulate: &v1beta2.AISimulateSpec{}}, - }}, - "empty object": runWithParameters(`{}`), - "not an object": runWithParameters(`[1]`), - "not json": runWithParameters(`{`), - } { - if _, err := InputForRun(run); err == nil { - t.Errorf("%s: want an error", name) - } - } -} diff --git a/deploy/operator/internal/controller/dynamographdeploymentrequest_v1beta2_controller.go b/deploy/operator/internal/controller/dynamographdeploymentrequest_v1beta2_controller.go index 605895009fa3..c88d2f1ec4af 100644 --- a/deploy/operator/internal/controller/dynamographdeploymentrequest_v1beta2_controller.go +++ b/deploy/operator/internal/controller/dynamographdeploymentrequest_v1beta2_controller.go @@ -201,7 +201,7 @@ func (r *DynamoGraphDeploymentRequestV1beta2Reconciler) Reconcile( // buildRun resolves the optional inputs of the Request into an immutable Run spec. The effective // AISimulate input is frozen into aisimulate.parameters: that object is the complete resolved input. -// Run execution reads it with aisimulate.InputForRun and never lowers the Run again, because the +// Run execution decodes it directly and never lowers the Run again, because the // merge-patch deletions (null) that shaped it are not part of the stored result. func (r *DynamoGraphDeploymentRequestV1beta2Reconciler) buildRun( request *nvidiacomv1beta2.DynamoGraphDeploymentRequest, @@ -400,11 +400,14 @@ func dgdrV1beta2SplitRuns( return active, terminal, nil } -// requireDGDRV1beta2API fails when the cluster does not serve the v1beta2 Request and Run APIs. -// Without them the controller's caches never sync and the manager would hang at startup. +// requireDGDRV1beta2API fails when the cluster does not serve the v1beta2 Request, Run and +// Candidate APIs. Without them either the manager caches cannot sync or a Run's publisher +// cannot start. func requireDGDRV1beta2API(mgr ctrl.Manager) error { group, version := nvidiacomv1beta2.GroupVersion.Group, nvidiacomv1beta2.GroupVersion.Version - for _, kind := range []string{"DynamoGraphDeploymentRequest", "DynamoGraphDeploymentRun"} { + for _, kind := range []string{ + "DynamoGraphDeploymentRequest", "DynamoGraphDeploymentRun", "DynamoGraphDeploymentCandidate", + } { _, err := mgr.GetRESTMapper().RESTMapping(schema.GroupKind{Group: group, Kind: kind}, version) if meta.IsNoMatchError(err) { return fmt.Errorf("--dgdr-v1beta2-enabled is set but the cluster does not serve %s/%s %s; "+ diff --git a/deploy/operator/internal/controller/dynamographdeploymentrequest_v1beta2_controller_envtest_test.go b/deploy/operator/internal/controller/dynamographdeploymentrequest_v1beta2_controller_envtest_test.go index 113c6160a218..2de8042f8ce2 100644 --- a/deploy/operator/internal/controller/dynamographdeploymentrequest_v1beta2_controller_envtest_test.go +++ b/deploy/operator/internal/controller/dynamographdeploymentrequest_v1beta2_controller_envtest_test.go @@ -96,7 +96,7 @@ func TestDGDRV1beta2ControllerReactsToRequestAndRunEvents(t *testing.T) { g.Expect(first.Spec.Hardware).NotTo(BeNil()) g.Expect(first.Spec.AISimulate.Image).To(Equal(controllerAISimulateImage)) g.Expect(first.Spec.AISimulate.VersionOverride).To(Equal("1.2.0")) - input, err := aisimulate.InputForRun(first) + input, err := aisimulate.DecodeParameters(first.Spec.AISimulate.Parameters) g.Expect(err).NotTo(HaveOccurred()) g.Expect(input).To(HaveKey("search_space")) diff --git a/deploy/operator/internal/controller/dynamographdeploymentrequest_v1beta2_controller_test.go b/deploy/operator/internal/controller/dynamographdeploymentrequest_v1beta2_controller_test.go index 953eb8e1fa02..71735c93ef92 100644 --- a/deploy/operator/internal/controller/dynamographdeploymentrequest_v1beta2_controller_test.go +++ b/deploy/operator/internal/controller/dynamographdeploymentrequest_v1beta2_controller_test.go @@ -201,7 +201,7 @@ func TestV1beta2RequestControllerCreatesTheRunOfTheCurrentGeneration(t *testing. if got := run.Spec.AISimulate.VersionOverride; got != "0.13.0" { t.Fatalf("run versionOverride = %q, want the resolved 0.13.0", got) } - parameters, err := aisimulate.InputForRun(run) + parameters, err := aisimulate.DecodeParameters(run.Spec.AISimulate.Parameters) if err != nil { t.Fatalf("input of the run: %v", err) } @@ -608,7 +608,7 @@ func TestV1beta2RequestControllerKeepsNullDeletedKeysOutOfTheRunInput(t *testing if err != nil { t.Fatalf("build run: %v", err) } - input, err := aisimulate.InputForRun(run) + input, err := aisimulate.DecodeParameters(run.Spec.AISimulate.Parameters) if err != nil { t.Fatalf("input of the run: %v", err) } diff --git a/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller.go b/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller.go new file mode 100644 index 000000000000..01f111fc85e6 --- /dev/null +++ b/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller.go @@ -0,0 +1,665 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package controller + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "math" + "path" + "strconv" + "strings" + + configv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/config/v1alpha1" + nvidiacomv1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" + nvidiacomv1beta2 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta2" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/controller/aisimulate" + commoncontroller "github.com/ai-dynamo/dynamo/deploy/operator/internal/controller_common" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/dgdrrunpublisher" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/gpu" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + apiequality "k8s.io/apimachinery/pkg/api/equality" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/types" + "k8s.io/utils/ptr" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/yaml" +) + +const ( + dgdrRunJobControllerUIDIndex = ".metadata.dgdrRunControllerUID" + dgdrRunMaxJobNameLength = 63 + + dgdrRunInputKey = "input.yaml" + dgdrRunInputMountPath = "/etc/aisimulate" + dgdrRunInputPath = dgdrRunInputMountPath + "/" + dgdrRunInputKey + dgdrRunSnapshotPath = "/var/run/aisimulate/output" + dgdrRunServiceAccountMountPath = "/var/run/secrets/kubernetes.io/serviceaccount" + dgdrRunPublisherBinary = "/dgdr-run-publisher" + dgdrRunSweeperContainer = "sweeper" + dgdrRunPublisherContainer = "publisher" + dgdrRunInputVolume = "input" + dgdrRunSnapshotVolume = "snapshot" + dgdrRunServiceAccountVolume = "publisher-kube-api" + dgdrRunGPUsPerNodeAnnotation = "nvidia.com/dgdr-gpus-per-node" + dgdrRunSnapshotPathAnnotation = "nvidia.com/dgdr-snapshot-path" + dgdrRunResourceComponent = "aisimulate" + dgdrRunServiceAccountTokenTTL = int64(3600) + dgdrRunReasonPending = "JobPending" + dgdrRunReasonRunning = "JobRunning" + dgdrRunReasonSucceeded = "SearchSucceeded" + dgdrRunReasonSearchFailed = "SearchFailed" + dgdrRunReasonProtocolViolation = "ProtocolViolation" + dgdrRunReasonPublicationFailed = "PublicationFailed" + dgdrRunReasonJobFailed = "JobFailed" + dgdrRunMessagePending = "AISimulate Job is waiting to start" + dgdrRunMessageRunning = "AISimulate Job is running" + dgdrRunMessageSucceeded = "AISimulate search completed successfully" +) + +type dgdrRunHardwareDiscoverer interface { + Discover(context.Context, nvidiacomv1beta1.GPUSKUType) (*gpu.GPUInfo, error) +} + +// DynamoGraphDeploymentRunV1beta2Reconciler executes one immutable Run as an +// AISimulate Job. The publisher sidecar owns progress and candidate publication; +// this controller owns the Job reference, execution phase, timestamps and Completed. +type DynamoGraphDeploymentRunV1beta2Reconciler struct { + client.Client + Config *configv1alpha1.OperatorConfiguration + RuntimeConfig *commoncontroller.RuntimeConfig + Hardware dgdrRunHardwareDiscoverer + OperatorImage string + OperatorImagePullPolicy corev1.PullPolicy +} + +// +kubebuilder:rbac:groups=nvidia.com,resources=dynamographdeploymentruns,verbs=get;list;watch +// +kubebuilder:rbac:groups=nvidia.com,resources=dynamographdeploymentruns/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=batch,resources=jobs,verbs=get;list;watch;create +// +kubebuilder:rbac:groups="",resources=pods,verbs=get;list;watch +// +kubebuilder:rbac:groups="",resources=nodes,verbs=get;list;watch +// +kubebuilder:rbac:groups="",resources=configmaps;serviceaccounts,verbs=get;list;watch;create;update;patch +// +kubebuilder:rbac:groups=rbac.authorization.k8s.io,resources=roles;rolebindings,verbs=get;list;watch;create;update;patch + +func (r *DynamoGraphDeploymentRunV1beta2Reconciler) Reconcile( + ctx context.Context, + req ctrl.Request, +) (ctrl.Result, error) { + run := &nvidiacomv1beta2.DynamoGraphDeploymentRun{} + if err := r.Get(ctx, req.NamespacedName, run); err != nil { + return ctrl.Result{}, client.IgnoreNotFound(err) + } + if run.DeletionTimestamp != nil || meta.IsStatusConditionTrue(run.Status.Conditions, nvidiacomv1beta2.DGDRRunConditionCompleted) { + return ctrl.Result{}, nil + } + if run.Spec.AISimulate == nil || run.Spec.AISimulate.Image == "" { + return ctrl.Result{}, fmt.Errorf("Run %q has no spec.aisimulate.image", run.Name) + } + + jobs, err := r.ownedJobs(ctx, run) + if err != nil { + return ctrl.Result{}, err + } + if len(jobs) > 1 { + return ctrl.Result{}, fmt.Errorf("Run %q controls %d Jobs; expected at most one", run.Name, len(jobs)) + } + + var job *batchv1.Job + if len(jobs) == 1 { + job = &jobs[0] + } + resourceName := dgdrRunResourceName(run.Name) + if err := r.reconcilePublisherAccess(ctx, run, resourceName); err != nil { + return ctrl.Result{}, err + } + execution, err := r.reconcileInput(ctx, run, job, resourceName) + if err != nil { + return ctrl.Result{}, err + } + + if job == nil { + job = r.buildJob(run, resourceName, execution) + if err := controllerutil.SetControllerReference(run, job, r.Scheme()); err != nil { + return ctrl.Result{}, fmt.Errorf("set Job owner: %w", err) + } + if err := r.Create(ctx, job); err != nil { + // A cache miss followed by AlreadyExists is expected to heal on the next + // reconciliation once the informer observes the Job. Never bypass the cache. + return ctrl.Result{}, fmt.Errorf("create Job %q: %w", job.Name, err) + } + } + + return ctrl.Result{}, r.reconcileRunStatus(ctx, run, job) +} + +func (r *DynamoGraphDeploymentRunV1beta2Reconciler) ownedJobs( + ctx context.Context, + run *nvidiacomv1beta2.DynamoGraphDeploymentRun, +) ([]batchv1.Job, error) { + list := &batchv1.JobList{} + if err := r.List(ctx, list, client.InNamespace(run.Namespace), + client.MatchingFields{dgdrRunJobControllerUIDIndex: string(run.UID)}); err != nil { + return nil, fmt.Errorf("list Jobs of Run %q: %w", run.Name, err) + } + owned := make([]batchv1.Job, 0, len(list.Items)) + for i := range list.Items { + if metav1.IsControlledBy(&list.Items[i], run) { + owned = append(owned, list.Items[i]) + } + } + return owned, nil +} + +func (r *DynamoGraphDeploymentRunV1beta2Reconciler) reconcilePublisherAccess( + ctx context.Context, + run *nvidiacomv1beta2.DynamoGraphDeploymentRun, + name string, +) error { + sa := &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: run.Namespace}} + if _, err := controllerutil.CreateOrUpdate(ctx, r.Client, sa, func() error { + return controllerutil.SetControllerReference(run, sa, r.Scheme()) + }); err != nil { + return fmt.Errorf("reconcile ServiceAccount %q: %w", name, err) + } + + role := &rbacv1.Role{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: run.Namespace}} + if _, err := controllerutil.CreateOrUpdate(ctx, r.Client, role, func() error { + role.Rules = dgdrRunPublisherRules(run.Name) + return controllerutil.SetControllerReference(run, role, r.Scheme()) + }); err != nil { + return fmt.Errorf("reconcile Role %q: %w", name, err) + } + + binding := &rbacv1.RoleBinding{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: run.Namespace}} + if _, err := controllerutil.CreateOrUpdate(ctx, r.Client, binding, func() error { + binding.RoleRef = rbacv1.RoleRef{APIGroup: rbacv1.GroupName, Kind: "Role", Name: name} + binding.Subjects = []rbacv1.Subject{{Kind: "ServiceAccount", Name: name, Namespace: run.Namespace}} + return controllerutil.SetControllerReference(run, binding, r.Scheme()) + }); err != nil { + return fmt.Errorf("reconcile RoleBinding %q: %w", name, err) + } + return nil +} + +func dgdrRunPublisherRules(runName string) []rbacv1.PolicyRule { + return []rbacv1.PolicyRule{ + { + APIGroups: []string{nvidiacomv1beta2.GroupVersion.Group}, + Resources: []string{"dynamographdeploymentruns"}, + ResourceNames: []string{runName}, + Verbs: []string{"get", "list", "watch"}, + }, + { + APIGroups: []string{nvidiacomv1beta2.GroupVersion.Group}, + Resources: []string{"dynamographdeploymentruns/status"}, + ResourceNames: []string{runName}, + Verbs: []string{"get", "update", "patch"}, + }, + { + APIGroups: []string{nvidiacomv1beta2.GroupVersion.Group}, + Resources: []string{"dynamographdeploymentcandidates"}, + Verbs: []string{"get", "list", "watch", "create", "patch", "delete"}, + }, + { + APIGroups: []string{nvidiacomv1beta2.GroupVersion.Group}, + Resources: []string{"dynamographdeploymentcandidates/status"}, + Verbs: []string{"get", "update", "patch"}, + }, + { + APIGroups: []string{""}, + Resources: []string{"pods"}, + Verbs: []string{"get", "list", "watch"}, + }, + } +} + +func (r *DynamoGraphDeploymentRunV1beta2Reconciler) reconcileInput( + ctx context.Context, + run *nvidiacomv1beta2.DynamoGraphDeploymentRun, + job *batchv1.Job, + name string, +) (dgdrRunExecution, error) { + configMap := &corev1.ConfigMap{} + key := types.NamespacedName{Namespace: run.Namespace, Name: name} + err := r.Get(ctx, key, configMap) + if err == nil { + if !metav1.IsControlledBy(configMap, run) { + return dgdrRunExecution{}, fmt.Errorf("ConfigMap %q is not controlled by Run %q", name, run.Name) + } + gpusPerNode, parseErr := positiveAnnotation(configMap.Annotations, dgdrRunGPUsPerNodeAnnotation) + if parseErr != nil { + return dgdrRunExecution{}, fmt.Errorf("read ConfigMap %q: %w", name, parseErr) + } + execution := dgdrRunExecution{ + GPUsPerNode: gpusPerNode, + SnapshotPath: annotationOrDefault(configMap.Annotations, dgdrRunSnapshotPathAnnotation, dgdrRunSnapshotPath), + } + if job != nil { + jobGPUs, jobErr := positiveAnnotation(job.Annotations, dgdrRunGPUsPerNodeAnnotation) + if jobErr != nil { + return dgdrRunExecution{}, fmt.Errorf("read Job %q: %w", job.Name, jobErr) + } + if jobGPUs != gpusPerNode { + return dgdrRunExecution{}, fmt.Errorf("Job %q and ConfigMap %q disagree about GPUs per node", job.Name, name) + } + jobSnapshotPath := annotationOrDefault(job.Annotations, dgdrRunSnapshotPathAnnotation, dgdrRunSnapshotPath) + if jobSnapshotPath != execution.SnapshotPath { + return dgdrRunExecution{}, fmt.Errorf("Job %q and ConfigMap %q disagree about the snapshot path", job.Name, name) + } + } + return execution, nil + } + if !apierrors.IsNotFound(err) { + return dgdrRunExecution{}, fmt.Errorf("get ConfigMap %q: %w", name, err) + } + + execution := dgdrRunExecution{SnapshotPath: dgdrRunSnapshotPath} + if job != nil { + execution.GPUsPerNode, err = positiveAnnotation(job.Annotations, dgdrRunGPUsPerNodeAnnotation) + if err != nil { + return dgdrRunExecution{}, fmt.Errorf("read Job %q: %w", job.Name, err) + } + execution.SnapshotPath = annotationOrDefault(job.Annotations, dgdrRunSnapshotPathAnnotation, dgdrRunSnapshotPath) + } + input, err := aisimulate.DecodeParameters(run.Spec.AISimulate.Parameters) + if err != nil { + return dgdrRunExecution{}, err + } + if execution.GPUsPerNode == 0 { + if configured, found := nestedPositiveInt(input, "dgdr_run", "num_gpus_per_node"); found { + execution.GPUsPerNode = configured + } else { + execution.GPUsPerNode, err = r.discoverGPUsPerNode(ctx, input) + if err != nil { + return dgdrRunExecution{}, err + } + } + } + contents, snapshotPath, err := dgdrRunInput(run, input, execution.GPUsPerNode) + if err != nil { + return dgdrRunExecution{}, err + } + if job == nil { + execution.SnapshotPath = snapshotPath + } + + configMap = &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: run.Namespace, + Annotations: map[string]string{ + dgdrRunGPUsPerNodeAnnotation: strconv.Itoa(execution.GPUsPerNode), + dgdrRunSnapshotPathAnnotation: execution.SnapshotPath, + }, + }, + Immutable: ptr.To(true), + Data: map[string]string{dgdrRunInputKey: string(contents)}, + } + if err := controllerutil.SetControllerReference(run, configMap, r.Scheme()); err != nil { + return dgdrRunExecution{}, fmt.Errorf("set ConfigMap owner: %w", err) + } + if err := r.Create(ctx, configMap); err != nil { + return dgdrRunExecution{}, fmt.Errorf("create ConfigMap %q: %w", name, err) + } + return execution, nil +} + +func (r *DynamoGraphDeploymentRunV1beta2Reconciler) discoverGPUsPerNode( + ctx context.Context, + input map[string]any, +) (int, error) { + if r.Hardware == nil { + return 0, errors.New("discover GPUs per node: hardware discoverer is not configured") + } + sku, _, err := unstructured.NestedString(input, "search_space", "hardware_sku") + if err != nil { + return 0, fmt.Errorf("read search_space.hardware_sku: %w", err) + } + info, err := r.Hardware.Discover(ctx, nvidiacomv1beta1.GPUSKUType(sku)) + if err != nil { + return 0, fmt.Errorf("discover GPUs per node: %w", err) + } + if info == nil || info.GPUsPerNode < 1 { + return 0, errors.New("discover GPUs per node: discovery returned no positive GPU count") + } + return info.GPUsPerNode, nil +} + +func dgdrRunInput( + run *nvidiacomv1beta2.DynamoGraphDeploymentRun, + parameters map[string]any, + gpusPerNode int, +) ([]byte, string, error) { + if run.Spec.Recommendation == nil || run.Spec.Recommendation.MaxCandidates < 1 { + return nil, "", errors.New("spec.recommendation.maxCandidates must be defaulted before the Run is created") + } + underlay := map[string]any{"dgdr_run": map[string]any{ + "name": run.Name, + "snapshot_dir": dgdrRunSnapshotPath, + "namespace": run.Namespace, + "runtime_image": run.Spec.AISimulate.Image, + "derive_runtime_images": true, + "num_gpus_per_node": int64(gpusPerNode), + "max_candidates": int64(run.Spec.Recommendation.MaxCandidates), + }} + input := aisimulate.MergeParameters(underlay, parameters) + snapshotPath := dgdrRunSnapshotPath + if configured, found, err := unstructured.NestedString(input, "dgdr_run", "snapshot_dir"); err == nil && found && path.IsAbs(configured) { + snapshotPath = path.Clean(configured) + } + contents, err := yaml.Marshal(input) + if err != nil { + return nil, "", fmt.Errorf("encode AISimulate input: %w", err) + } + return contents, snapshotPath, nil +} + +type dgdrRunExecution struct { + GPUsPerNode int + SnapshotPath string +} + +func nestedPositiveInt(input map[string]any, fields ...string) (int, bool) { + value, found, err := unstructured.NestedFieldNoCopy(input, fields...) + if err != nil || !found { + return 0, false + } + var number float64 + switch typed := value.(type) { + case int64: + number = float64(typed) + case float64: + number = typed + default: + return 0, false + } + if number < 1 || number > math.MaxInt || math.Trunc(number) != number { + return 0, false + } + return int(number), true +} + +func positiveAnnotation(annotations map[string]string, key string) (int, error) { + value := annotations[key] + parsed, err := strconv.Atoi(value) + if err != nil || parsed < 1 { + return 0, fmt.Errorf("annotation %s=%q is not a positive integer", key, value) + } + return parsed, nil +} + +func annotationOrDefault(annotations map[string]string, key, defaultValue string) string { + if value := annotations[key]; value != "" { + return value + } + return defaultValue +} + +func (r *DynamoGraphDeploymentRunV1beta2Reconciler) buildJob( + run *nvidiacomv1beta2.DynamoGraphDeploymentRun, + name string, + execution dgdrRunExecution, +) *batchv1.Job { + pullPolicy := r.OperatorImagePullPolicy + if pullPolicy == "" { + pullPolicy = corev1.PullIfNotPresent + } + job := &batchv1.Job{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: run.Namespace, + Annotations: map[string]string{ + dgdrRunGPUsPerNodeAnnotation: strconv.Itoa(execution.GPUsPerNode), + dgdrRunSnapshotPathAnnotation: execution.SnapshotPath, + }, + }, + Spec: batchv1.JobSpec{ + BackoffLimit: ptr.To[int32](0), + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + ServiceAccountName: name, + AutomountServiceAccountToken: ptr.To(false), + RestartPolicy: corev1.RestartPolicyNever, + SecurityContext: &corev1.PodSecurityContext{FSGroup: ptr.To[int64](65532)}, + Containers: []corev1.Container{ + { + Name: dgdrRunSweeperContainer, + Image: run.Spec.AISimulate.Image, + ImagePullPolicy: corev1.PullIfNotPresent, + Command: []string{"aisimulate"}, + Args: []string{ + "recommend", "--stack", "dynamo", "--config", dgdrRunInputPath, + "--output", "dgdr_run", "--output-dir", execution.SnapshotPath, + }, + VolumeMounts: []corev1.VolumeMount{ + {Name: dgdrRunInputVolume, MountPath: dgdrRunInputMountPath, ReadOnly: true}, + {Name: dgdrRunSnapshotVolume, MountPath: execution.SnapshotPath}, + }, + }, + { + Name: dgdrRunPublisherContainer, + Image: r.OperatorImage, + ImagePullPolicy: pullPolicy, + Command: []string{dgdrRunPublisherBinary}, + Args: []string{ + "--snapshot-dir", execution.SnapshotPath, + "--run-name", run.Name, + "--run-uid", string(run.UID), + "--sweeper-container", dgdrRunSweeperContainer, + }, + Env: []corev1.EnvVar{ + {Name: "POD_NAMESPACE", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.namespace"}}}, + {Name: "POD_NAME", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.name"}}}, + }, + VolumeMounts: []corev1.VolumeMount{ + {Name: dgdrRunSnapshotVolume, MountPath: execution.SnapshotPath}, + {Name: dgdrRunServiceAccountVolume, MountPath: dgdrRunServiceAccountMountPath, ReadOnly: true}, + }, + }, + }, + Volumes: []corev1.Volume{ + { + Name: dgdrRunInputVolume, + VolumeSource: corev1.VolumeSource{ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{Name: name}, + }}, + }, + {Name: dgdrRunSnapshotVolume, VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}}, + dgdrRunPublisherKubeAPIVolume(), + }, + }, + }, + }, + } + return job +} + +func dgdrRunPublisherKubeAPIVolume() corev1.Volume { + return corev1.Volume{ + Name: dgdrRunServiceAccountVolume, + VolumeSource: corev1.VolumeSource{Projected: &corev1.ProjectedVolumeSource{ + Sources: []corev1.VolumeProjection{ + {ServiceAccountToken: &corev1.ServiceAccountTokenProjection{ + Path: "token", ExpirationSeconds: ptr.To(dgdrRunServiceAccountTokenTTL), + }}, + {ConfigMap: &corev1.ConfigMapProjection{ + LocalObjectReference: corev1.LocalObjectReference{Name: ConfigMapNameKubeRootCA}, + Items: []corev1.KeyToPath{{Key: "ca.crt", Path: "ca.crt"}}, + }}, + {DownwardAPI: &corev1.DownwardAPIProjection{Items: []corev1.DownwardAPIVolumeFile{{ + Path: "namespace", FieldRef: &corev1.ObjectFieldSelector{APIVersion: "v1", FieldPath: "metadata.namespace"}, + }}}}, + }, + }}, + } +} + +func (r *DynamoGraphDeploymentRunV1beta2Reconciler) reconcileRunStatus( + ctx context.Context, + run *nvidiacomv1beta2.DynamoGraphDeploymentRun, + job *batchv1.Job, +) error { + original := run.DeepCopy() + run.Status.JobRef = &nvidiacomv1beta2.ResourceReference{Name: job.Name, UID: job.UID} + run.Status.Phase = nvidiacomv1beta2.DGDRRunPhasePending + reason, message := dgdrRunReasonPending, dgdrRunMessagePending + completed := metav1.ConditionFalse + + if job.Status.StartTime != nil { + run.Status.StartTime = job.Status.StartTime.DeepCopy() + } + if condition := dgdrRunJobCondition(job, batchv1.JobComplete); condition != nil && condition.Status == corev1.ConditionTrue { + run.Status.Phase = nvidiacomv1beta2.DGDRRunPhaseSucceeded + reason, message = dgdrRunReasonSucceeded, dgdrRunMessageSucceeded + completed = metav1.ConditionTrue + run.Status.CompletionTime = dgdrRunCompletionTime(job) + } else if condition := dgdrRunJobCondition(job, batchv1.JobFailed); condition != nil && condition.Status == corev1.ConditionTrue { + run.Status.Phase = nvidiacomv1beta2.DGDRRunPhaseFailed + reason, message = r.dgdrRunFailure(ctx, job, condition) + completed = metav1.ConditionTrue + run.Status.CompletionTime = dgdrRunCompletionTime(job) + } else if job.Status.Active > 0 || job.Status.StartTime != nil { + run.Status.Phase = nvidiacomv1beta2.DGDRRunPhaseRunning + reason, message = dgdrRunReasonRunning, dgdrRunMessageRunning + } + + meta.SetStatusCondition(&run.Status.Conditions, metav1.Condition{ + Type: nvidiacomv1beta2.DGDRRunConditionCompleted, + Status: completed, + Reason: reason, + Message: message, + ObservedGeneration: run.Generation, + }) + if apiequality.Semantic.DeepEqual(original.Status, run.Status) { + return nil + } + if err := r.Status().Patch(ctx, run, client.MergeFrom(original)); err != nil { + return fmt.Errorf("update Run %q status: %w", run.Name, err) + } + return nil +} + +func (r *DynamoGraphDeploymentRunV1beta2Reconciler) dgdrRunFailure( + ctx context.Context, + job *batchv1.Job, + condition *batchv1.JobCondition, +) (string, string) { + pods := &corev1.PodList{} + if err := r.List(ctx, pods, client.InNamespace(job.Namespace), client.MatchingLabels{batchv1.JobNameLabel: job.Name}); err != nil { + log.FromContext(ctx).Error(err, "Could not inspect failed AISimulate Job pods", "job", job.Name) + return dgdrRunReasonJobFailed, condition.Message + } + for i := range pods.Items { + pod := &pods.Items[i] + if !metav1.IsControlledBy(pod, job) { + continue + } + for _, status := range pod.Status.ContainerStatuses { + terminated := status.State.Terminated + if terminated == nil || terminated.ExitCode == 0 { + continue + } + message := terminated.Message + if message == "" { + message = terminated.Reason + } + if message == "" { + message = fmt.Sprintf("container %s exited with code %d", status.Name, terminated.ExitCode) + } + if status.Name == dgdrRunPublisherContainer { + switch int(terminated.ExitCode) { + case dgdrrunpublisher.ExitRunFailed: + return dgdrRunReasonSearchFailed, message + case dgdrrunpublisher.ExitProtocolViolation: + return dgdrRunReasonProtocolViolation, message + case dgdrrunpublisher.ExitReconcileFailed: + return dgdrRunReasonPublicationFailed, message + } + } + if status.Name == dgdrRunSweeperContainer { + return dgdrRunReasonSearchFailed, message + } + } + } + message := condition.Message + if message == "" { + message = "AISimulate Job failed" + } + return dgdrRunReasonJobFailed, message +} + +func dgdrRunCompletionTime(job *batchv1.Job) *metav1.Time { + if job.Status.CompletionTime != nil { + return job.Status.CompletionTime.DeepCopy() + } + for _, condition := range job.Status.Conditions { + if (condition.Type == batchv1.JobComplete || condition.Type == batchv1.JobFailed) && condition.Status == corev1.ConditionTrue { + return condition.LastTransitionTime.DeepCopy() + } + } + return nil +} + +func dgdrRunJobCondition(job *batchv1.Job, conditionType batchv1.JobConditionType) *batchv1.JobCondition { + for i := range job.Status.Conditions { + if job.Status.Conditions[i].Type == conditionType { + return &job.Status.Conditions[i] + } + } + return nil +} + +func dgdrRunResourceName(runName string) string { + suffix := "-" + dgdrRunResourceComponent + if len(runName)+len(suffix) <= dgdrRunMaxJobNameLength { + return runName + suffix + } + sum := sha256.Sum256([]byte(runName)) + hash := hex.EncodeToString(sum[:6]) + room := dgdrRunMaxJobNameLength - len(suffix) - len(hash) - 1 + prefix := strings.TrimRight(runName[:room], "-.") + return prefix + "-" + hash + suffix +} + +func (r *DynamoGraphDeploymentRunV1beta2Reconciler) SetupWithManager(mgr ctrl.Manager) error { + if r.OperatorImage == "" { + return errors.New("operator image is required for the DGDR Run publisher") + } + if err := mgr.GetFieldIndexer().IndexField( + context.Background(), &batchv1.Job{}, dgdrRunJobControllerUIDIndex, dgdrRunJobControllerUIDs, + ); err != nil { + return fmt.Errorf("index Jobs by Run controller UID: %w", err) + } + return ctrl.NewControllerManagedBy(mgr). + Named("dynamographdeploymentrun-v1beta2"). + For(&nvidiacomv1beta2.DynamoGraphDeploymentRun{}). + Owns(&batchv1.Job{}). + Owns(&corev1.ConfigMap{}). + Owns(&corev1.ServiceAccount{}). + Owns(&rbacv1.Role{}). + Owns(&rbacv1.RoleBinding{}). + WithEventFilter(commoncontroller.EphemeralDeploymentEventFilter(r.Config, r.RuntimeConfig)). + Complete(r) +} + +func dgdrRunJobControllerUIDs(obj client.Object) []string { + owner := metav1.GetControllerOf(obj) + if owner == nil || owner.Kind != "DynamoGraphDeploymentRun" || owner.APIVersion != nvidiacomv1beta2.GroupVersion.String() { + return nil + } + return []string{string(owner.UID)} +} diff --git a/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller_test.go b/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller_test.go new file mode 100644 index 000000000000..f53949022e53 --- /dev/null +++ b/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller_test.go @@ -0,0 +1,392 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package controller + +import ( + "context" + "encoding/json" + "reflect" + "strings" + "testing" + "time" + + nvidiacomv1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" + nvidiacomv1beta2 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta2" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/dgdrrunpublisher" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/gpu" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + "k8s.io/utils/ptr" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/yaml" +) + +type fixedRunHardware struct { + info gpu.GPUInfo + err error + calls []nvidiacomv1beta1.GPUSKUType +} + +func (f *fixedRunHardware) Discover(_ context.Context, sku nvidiacomv1beta1.GPUSKUType) (*gpu.GPUInfo, error) { + f.calls = append(f.calls, sku) + if f.err != nil { + return nil, f.err + } + return &f.info, nil +} + +type dgdrRunFixture struct { + t *testing.T + client client.Client + reconciler *DynamoGraphDeploymentRunV1beta2Reconciler +} + +func newDGDRRunFixture(t *testing.T, hardware dgdrRunHardwareDiscoverer, objects ...client.Object) *dgdrRunFixture { + t.Helper() + scheme := runtime.NewScheme() + if err := clientgoscheme.AddToScheme(scheme); err != nil { + t.Fatalf("add Kubernetes types: %v", err) + } + if err := nvidiacomv1beta2.AddToScheme(scheme); err != nil { + t.Fatalf("add v1beta2 types: %v", err) + } + c := fake.NewClientBuilder(). + WithScheme(scheme). + WithObjects(objects...). + WithIndex(&batchv1.Job{}, dgdrRunJobControllerUIDIndex, dgdrRunJobControllerUIDs). + WithStatusSubresource(&nvidiacomv1beta2.DynamoGraphDeploymentRun{}, &batchv1.Job{}, &corev1.Pod{}). + Build() + return &dgdrRunFixture{ + t: t, + client: c, + reconciler: &DynamoGraphDeploymentRunV1beta2Reconciler{ + Client: c, + Hardware: hardware, + OperatorImage: "example.com/dynamo-operator:1.2.3", + OperatorImagePullPolicy: corev1.PullAlways, + }, + } +} + +func (f *dgdrRunFixture) reconcile() { + f.t.Helper() + if _, err := f.reconciler.Reconcile(f.t.Context(), ctrl.Request{NamespacedName: types.NamespacedName{ + Namespace: "default", Name: "search-1", + }}); err != nil { + f.t.Fatalf("reconcile: %v", err) + } +} + +func (f *dgdrRunFixture) run() *nvidiacomv1beta2.DynamoGraphDeploymentRun { + f.t.Helper() + run := &nvidiacomv1beta2.DynamoGraphDeploymentRun{} + if err := f.client.Get(f.t.Context(), types.NamespacedName{Namespace: "default", Name: "search-1"}, run); err != nil { + f.t.Fatalf("get Run: %v", err) + } + return run +} + +func (f *dgdrRunFixture) job() *batchv1.Job { + f.t.Helper() + jobs := &batchv1.JobList{} + if err := f.client.List(f.t.Context(), jobs, client.InNamespace("default")); err != nil { + f.t.Fatalf("list Jobs: %v", err) + } + if len(jobs.Items) != 1 { + f.t.Fatalf("Jobs = %d, want 1", len(jobs.Items)) + } + return &jobs.Items[0] +} + +func dgdrRunForTest(t *testing.T, parameters map[string]any) *nvidiacomv1beta2.DynamoGraphDeploymentRun { + t.Helper() + raw, err := json.Marshal(parameters) + if err != nil { + t.Fatalf("marshal parameters: %v", err) + } + return &nvidiacomv1beta2.DynamoGraphDeploymentRun{ + ObjectMeta: metav1.ObjectMeta{Name: "search-1", Namespace: "default", UID: "run-uid", Generation: 1}, + Spec: nvidiacomv1beta2.DynamoGraphDeploymentRunSpec{NativeSearchInput: nvidiacomv1beta2.NativeSearchInput{ + Model: &nvidiacomv1beta2.ModelSpec{Name: "Qwen/Qwen3-0.6B"}, + AISimulate: &nvidiacomv1beta2.AISimulateSpec{Image: "example.com/dynamo-planner:1.2.3", Parameters: &runtime.RawExtension{Raw: raw}}, + Recommendation: &nvidiacomv1beta2.RecommendationSpec{MaxCandidates: 2}, + }}, + } +} + +func TestV1beta2RunControllerCreatesOneIsolatedAISimulateJob(t *testing.T) { + hardware := &fixedRunHardware{info: gpu.GPUInfo{GPUsPerNode: 8}} + run := dgdrRunForTest(t, map[string]any{ + "search_space": map[string]any{"hardware_sku": "h200_sxm", "gpu_budget": 16}, + "dgdr_run": map[string]any{ + "max_candidates": 7, + "runtime_image": "example.com/custom-vllm-runtime:1.2.3", + "name": "native-output-name", + "snapshot_dir": "/work/results", + }, + }) + f := newDGDRRunFixture(t, hardware, run) + + f.reconcile() + f.reconcile() + + job := f.job() + assertDGDRRunJobIsolation(t, job, run, f.reconciler.OperatorImage) + assertDGDRRunInput(t, f.client, job, hardware) + assertDGDRRunSupportResources(t, f.client, job, run) + assertDGDRRunPendingStatus(t, f.run(), job.Name) +} + +func assertDGDRRunJobIsolation( + t *testing.T, + job *batchv1.Job, + run *nvidiacomv1beta2.DynamoGraphDeploymentRun, + operatorImage string, +) { + t.Helper() + if !metav1.IsControlledBy(job, run) { + t.Fatal("Job is not controlled by its Run") + } + if got := job.Annotations[dgdrRunGPUsPerNodeAnnotation]; got != "8" { + t.Fatalf("GPUs-per-node annotation = %q, want 8", got) + } + if job.Spec.BackoffLimit == nil || *job.Spec.BackoffLimit != 0 { + t.Fatalf("backoffLimit = %v, want 0", job.Spec.BackoffLimit) + } + if job.Spec.Template.Spec.AutomountServiceAccountToken == nil || *job.Spec.Template.Spec.AutomountServiceAccountToken { + t.Fatal("the pod must disable automatic service-account token mounting") + } + sweeper := containerNamed(t, job.Spec.Template.Spec.Containers, dgdrRunSweeperContainer) + publisher := containerNamed(t, job.Spec.Template.Spec.Containers, dgdrRunPublisherContainer) + if sweeper.Image != run.Spec.AISimulate.Image { + t.Fatalf("sweeper image = %q, want %q", sweeper.Image, run.Spec.AISimulate.Image) + } + if publisher.Image != operatorImage || publisher.ImagePullPolicy != corev1.PullAlways { + t.Fatalf("publisher image = %q/%s, want configured operator image/Always", publisher.Image, publisher.ImagePullPolicy) + } + if hasMount(sweeper.VolumeMounts, dgdrRunServiceAccountVolume) { + t.Fatal("sweeper unexpectedly receives the Kubernetes service-account token") + } + if !hasMount(publisher.VolumeMounts, dgdrRunServiceAccountVolume) { + t.Fatal("publisher does not receive its projected Kubernetes credentials") + } + if !hasMountAt(sweeper.VolumeMounts, dgdrRunSnapshotVolume, "/work/results") || + !hasMountAt(publisher.VolumeMounts, dgdrRunSnapshotVolume, "/work/results") { + t.Fatal("sweeper and publisher do not share the parameter-overridden snapshot directory") + } +} + +func assertDGDRRunInput(t *testing.T, c client.Client, job *batchv1.Job, hardware *fixedRunHardware) { + t.Helper() + configMap := &corev1.ConfigMap{} + if err := c.Get(t.Context(), client.ObjectKeyFromObject(job), configMap); err != nil { + t.Fatalf("get input ConfigMap: %v", err) + } + if configMap.Immutable == nil || !*configMap.Immutable { + t.Fatal("input ConfigMap is not immutable") + } + var input map[string]any + if err := yaml.Unmarshal([]byte(configMap.Data[dgdrRunInputKey]), &input); err != nil { + t.Fatalf("decode input ConfigMap: %v", err) + } + output := input["dgdr_run"].(map[string]any) + if output["name"] != "native-output-name" || output["snapshot_dir"] != "/work/results" { + t.Fatalf("native parameter overrides were not preserved: %v", output) + } + if output["max_candidates"] != float64(7) || output["runtime_image"] != "example.com/custom-vllm-runtime:1.2.3" { + t.Fatalf("native parameter overrides were not preserved: %v", output) + } + if output["derive_runtime_images"] != true { + t.Fatalf("derive_runtime_images = %v, want true for a backend search", output["derive_runtime_images"]) + } + if got := input["search_space"].(map[string]any)["hardware_sku"]; got != "h200_sxm" { + t.Fatalf("search_space.hardware_sku = %v, want h200_sxm", got) + } + if !reflect.DeepEqual(hardware.calls, []nvidiacomv1beta1.GPUSKUType{"h200_sxm"}) { + t.Fatalf("hardware discovery calls = %v, want one filtered call", hardware.calls) + } +} + +func assertDGDRRunSupportResources( + t *testing.T, + c client.Client, + job *batchv1.Job, + run *nvidiacomv1beta2.DynamoGraphDeploymentRun, +) { + t.Helper() + for _, object := range []client.Object{ + &corev1.ServiceAccount{}, &rbacv1.Role{}, &rbacv1.RoleBinding{}, + } { + if err := c.Get(t.Context(), client.ObjectKeyFromObject(job), object); err != nil { + t.Fatalf("get %T: %v", object, err) + } + if !metav1.IsControlledBy(object, run) { + t.Fatalf("%T is not controlled by the Run", object) + } + } +} + +func assertDGDRRunPendingStatus(t *testing.T, stored *nvidiacomv1beta2.DynamoGraphDeploymentRun, jobName string) { + t.Helper() + if stored.Status.Phase != nvidiacomv1beta2.DGDRRunPhasePending || stored.Status.JobRef == nil || stored.Status.JobRef.Name != jobName { + t.Fatalf("Run status = %+v, want Pending and JobRef %q", stored.Status, jobName) + } + condition := meta.FindStatusCondition(stored.Status.Conditions, nvidiacomv1beta2.DGDRRunConditionCompleted) + if condition == nil || condition.Status != metav1.ConditionFalse || condition.Reason != dgdrRunReasonPending { + t.Fatalf("Completed condition = %+v, want False/%s", condition, dgdrRunReasonPending) + } +} + +func TestV1beta2RunControllerOwnsOnlyLifecycleStatus(t *testing.T) { + hardware := &fixedRunHardware{info: gpu.GPUInfo{GPUsPerNode: 8}} + f := newDGDRRunFixture(t, hardware, dgdrRunForTest(t, map[string]any{ + "search_space": map[string]any{"hardware_sku": "h200_sxm", "gpu_budget": 8}, + })) + f.reconcile() + job := f.job() + + run := f.run() + progressTime := metav1.NewTime(time.Date(2026, 10, 23, 10, 0, 0, 0, time.UTC)) + run.Status.Progress = &nvidiacomv1beta2.DGDRRunProgress{Candidates: &nvidiacomv1beta2.CandidateProgress{Published: 1}} + run.Status.CandidateRefs = []corev1.LocalObjectReference{{Name: "candidate-a"}} + run.Status.LastProgressTime = &progressTime + if err := f.client.Status().Update(t.Context(), run); err != nil { + t.Fatalf("publish progress status: %v", err) + } + + start := metav1.NewTime(progressTime.Add(time.Minute)) + job.Status.StartTime = &start + job.Status.Active = 1 + if err := f.client.Status().Update(t.Context(), job); err != nil { + t.Fatalf("mark Job running: %v", err) + } + f.reconcile() + run = f.run() + if run.Status.Phase != nvidiacomv1beta2.DGDRRunPhaseRunning || run.Status.StartTime == nil || !run.Status.StartTime.Equal(&start) { + t.Fatalf("running status = %+v", run.Status) + } + if run.Status.Progress == nil || len(run.Status.CandidateRefs) != 1 || run.Status.LastProgressTime == nil { + t.Fatalf("publisher-owned status was lost: %+v", run.Status) + } + + completed := metav1.NewTime(start.Add(5 * time.Minute)) + job = f.job() + job.Status.Active = 0 + job.Status.CompletionTime = &completed + job.Status.Conditions = []batchv1.JobCondition{{ + Type: batchv1.JobComplete, Status: corev1.ConditionTrue, LastTransitionTime: completed, + }} + if err := f.client.Status().Update(t.Context(), job); err != nil { + t.Fatalf("complete Job: %v", err) + } + f.reconcile() + run = f.run() + condition := meta.FindStatusCondition(run.Status.Conditions, nvidiacomv1beta2.DGDRRunConditionCompleted) + if run.Status.Phase != nvidiacomv1beta2.DGDRRunPhaseSucceeded || condition == nil || condition.Status != metav1.ConditionTrue { + t.Fatalf("completed status = %+v", run.Status) + } + if run.Status.CompletionTime == nil || !run.Status.CompletionTime.Equal(&completed) { + t.Fatalf("completionTime = %v, want %v", run.Status.CompletionTime, completed) + } +} + +func TestV1beta2RunControllerClassifiesPublisherProtocolFailure(t *testing.T) { + f := newDGDRRunFixture(t, &fixedRunHardware{info: gpu.GPUInfo{GPUsPerNode: 8}}, dgdrRunForTest(t, map[string]any{ + "search_space": map[string]any{"hardware_sku": "h200_sxm", "gpu_budget": 8}, + })) + f.reconcile() + job := f.job() + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: job.Name + "-pod", + Namespace: job.Namespace, + Labels: map[string]string{batchv1.JobNameLabel: job.Name}, + OwnerReferences: []metav1.OwnerReference{{ + APIVersion: batchv1.SchemeGroupVersion.String(), Kind: "Job", Name: job.Name, UID: job.UID, + Controller: ptr.To(true), + }}, + }, + } + if err := f.client.Create(t.Context(), pod); err != nil { + t.Fatalf("create Job pod: %v", err) + } + pod.Status.ContainerStatuses = []corev1.ContainerStatus{{ + Name: dgdrRunPublisherContainer, + State: corev1.ContainerState{Terminated: &corev1.ContainerStateTerminated{ + ExitCode: int32(dgdrrunpublisher.ExitProtocolViolation), Message: "terminal snapshot missing", + }}, + }} + if err := f.client.Status().Update(t.Context(), pod); err != nil { + t.Fatalf("update Job pod status: %v", err) + } + + failed := metav1.Now() + job.Status.Conditions = []batchv1.JobCondition{{ + Type: batchv1.JobFailed, Status: corev1.ConditionTrue, LastTransitionTime: failed, + }} + if err := f.client.Status().Update(t.Context(), job); err != nil { + t.Fatalf("fail Job: %v", err) + } + f.reconcile() + + run := f.run() + condition := meta.FindStatusCondition(run.Status.Conditions, nvidiacomv1beta2.DGDRRunConditionCompleted) + if run.Status.Phase != nvidiacomv1beta2.DGDRRunPhaseFailed || condition == nil || condition.Reason != dgdrRunReasonProtocolViolation { + t.Fatalf("failed status = %+v", run.Status) + } + if !strings.Contains(condition.Message, "terminal snapshot missing") { + t.Fatalf("failure message = %q, want publisher termination message", condition.Message) + } +} + +func TestDGDRRunResourceNameIsStableAndBounded(t *testing.T) { + short := dgdrRunResourceName("search-1") + if short != "search-1-aisimulate" { + t.Fatalf("short name = %q", short) + } + long := strings.Repeat("a", 200) + one, two := dgdrRunResourceName(long), dgdrRunResourceName(long) + if one != two || len(one) > dgdrRunMaxJobNameLength || !strings.HasSuffix(one, "-aisimulate") { + t.Fatalf("bounded name = %q", one) + } +} + +func containerNamed(t *testing.T, containers []corev1.Container, name string) corev1.Container { + t.Helper() + for _, container := range containers { + if container.Name == name { + return container + } + } + t.Fatalf("container %q not found", name) + return corev1.Container{} +} + +func hasMount(mounts []corev1.VolumeMount, name string) bool { + for _, mount := range mounts { + if mount.Name == name { + return true + } + } + return false +} + +func hasMountAt(mounts []corev1.VolumeMount, name, path string) bool { + for _, mount := range mounts { + if mount.Name == name && mount.MountPath == path { + return true + } + } + return false +} diff --git a/deploy/operator/internal/controller/setup.go b/deploy/operator/internal/controller/setup.go index 42a081839153..94e96f1ac4bb 100644 --- a/deploy/operator/internal/controller/setup.go +++ b/deploy/operator/internal/controller/setup.go @@ -11,6 +11,7 @@ import ( configv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/config/v1alpha1" lpxcontroller "github.com/ai-dynamo/dynamo/deploy/operator/internal/controller/lpx" commoncontroller "github.com/ai-dynamo/dynamo/deploy/operator/internal/controller_common" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/features" "github.com/ai-dynamo/dynamo/deploy/operator/internal/gpu" "github.com/ai-dynamo/dynamo/deploy/operator/internal/modelendpoint" "github.com/ai-dynamo/dynamo/deploy/operator/internal/secret" @@ -44,6 +45,14 @@ type DynamoGraphDeploymentRequestSetupOptions struct { OperatorImagePullPolicy corev1.PullPolicy } +type DynamoGraphDeploymentRunSetupOptions struct { + SetupOptions + GPUDiscoveryCache *gpu.GPUDiscoveryCache + GPUDiscovery *gpu.GPUDiscovery + OperatorImage string + OperatorImagePullPolicy corev1.PullPolicy +} + type DynamoModelSetupOptions struct { SetupOptions ModelEndpointClient *modelendpoint.Client @@ -63,6 +72,20 @@ func (o DynamoGraphDeploymentRequestSetupOptions) gpuDiscovery() *gpu.GPUDiscove return gpu.NewGPUDiscovery(gpu.ScrapeMetricsEndpoint) } +func (o DynamoGraphDeploymentRunSetupOptions) gpuDiscoveryCache() *gpu.GPUDiscoveryCache { + if o.GPUDiscoveryCache != nil { + return o.GPUDiscoveryCache + } + return gpu.NewGPUDiscoveryCache() +} + +func (o DynamoGraphDeploymentRunSetupOptions) gpuDiscovery() *gpu.GPUDiscovery { + if o.GPUDiscovery != nil { + return o.GPUDiscovery + } + return gpu.NewGPUDiscovery(gpu.ScrapeMetricsEndpoint) +} + func (o DynamoModelSetupOptions) modelEndpointClient() *modelendpoint.Client { if o.ModelEndpointClient != nil { return o.ModelEndpointClient @@ -149,6 +172,29 @@ func SetupDynamoGraphDeploymentRequestV1beta2(mgr ctrl.Manager, opts SetupOption return nil } +// SetupDynamoGraphDeploymentRunV1beta2 registers the controller that executes immutable +// v1beta2 Runs as AISimulate Jobs and observes their lifecycle. +func SetupDynamoGraphDeploymentRunV1beta2(mgr ctrl.Manager, opts DynamoGraphDeploymentRunSetupOptions) error { + if err := (&DynamoGraphDeploymentRunV1beta2Reconciler{ + Client: mgr.GetClient(), + Config: opts.Config, + RuntimeConfig: opts.RuntimeConfig, + Hardware: &gpu.Discoverer{ + Reader: mgr.GetClient(), + DCGM: opts.gpuDiscovery(), + Cache: opts.gpuDiscoveryCache(), + NodeLabels: func() bool { + return opts.RuntimeConfig != nil && opts.RuntimeConfig.Gate.Enabled(features.GPUDiscovery) + }, + }, + OperatorImage: opts.OperatorImage, + OperatorImagePullPolicy: opts.OperatorImagePullPolicy, + }).SetupWithManager(mgr); err != nil { + return fmt.Errorf("unable to create DynamoGraphDeploymentRun v1beta2 controller: %w", err) + } + return nil +} + func SetupDynamoModel(mgr ctrl.Manager, opts DynamoModelSetupOptions) error { if err := (&DynamoModelReconciler{ Client: mgr.GetClient(), From 7e4ec97277d7fcbb3e1ebd556db2765a16dcc42a Mon Sep 17 00:00:00 2001 From: "Dr. Stefan Schimanski" Date: Mon, 26 Oct 2026 14:50:27 +0100 Subject: [PATCH 40/40] operator/dgdr: preserve runtime version for planner jobs Signed-off-by: Dr. Stefan Schimanski --- deploy/operator/cmd/main.go | 8 ++- ...mographdeploymentrun_v1beta2_controller.go | 16 ++++-- ...phdeploymentrun_v1beta2_controller_test.go | 50 +++++++++++++++++++ deploy/operator/internal/controller/setup.go | 2 + 4 files changed, 71 insertions(+), 5 deletions(-) diff --git a/deploy/operator/cmd/main.go b/deploy/operator/cmd/main.go index a3ab590bd786..723ed748e6d3 100644 --- a/deploy/operator/cmd/main.go +++ b/deploy/operator/cmd/main.go @@ -201,11 +201,13 @@ func main() { setupLog.Info("Operator configuration loaded successfully", "configFile", configFile) // Validate and normalize operator version to semver - if _, err := semver.NewVersion(operatorVersion); err != nil { + parsedOperatorVersion, err := semver.NewVersion(operatorVersion) + if err != nil { setupLog.Error(err, "operator-version is not valid semver", "provided", operatorVersion, "error", err.Error()) os.Exit(1) } + dynamoVersion := fmt.Sprintf("%d.%d.%d", parsedOperatorVersion.Major(), parsedOperatorVersion.Minor(), parsedOperatorVersion.Patch()) setupLog.Info("Operator version configured", "version", operatorVersion) pullPolicy := corev1.PullPolicy(operatorImagePullPolicy) @@ -506,7 +508,7 @@ func main() { if err := registerControllers( mgr, operatorCfg, runtimeConfig, dockerSecretRetriever, sshKeyManager, - operatorImage, pullPolicy, + operatorImage, pullPolicy, dynamoVersion, dgdrV1beta2Enabled, ); err != nil { setupLog.Error(err, "failed to register controllers") @@ -557,6 +559,7 @@ func registerControllers( sshKeyManager *secret.SSHKeyManager, operatorImage string, operatorPullPolicy corev1.PullPolicy, + dynamoVersion string, dgdrV1beta2Enabled bool, ) error { if operatorCfg.Namespace.Restricted == "" { @@ -628,6 +631,7 @@ func registerControllers( } if err := controller.SetupDynamoGraphDeploymentRunV1beta2(mgr, controller.DynamoGraphDeploymentRunSetupOptions{ SetupOptions: setupOptions, + DynamoVersion: dynamoVersion, OperatorImage: operatorImage, OperatorImagePullPolicy: operatorPullPolicy, }); err != nil { diff --git a/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller.go b/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller.go index 01f111fc85e6..d1aa832f20de 100644 --- a/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller.go +++ b/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller.go @@ -23,6 +23,7 @@ import ( commoncontroller "github.com/ai-dynamo/dynamo/deploy/operator/internal/controller_common" "github.com/ai-dynamo/dynamo/deploy/operator/internal/dgdrrunpublisher" "github.com/ai-dynamo/dynamo/deploy/operator/internal/gpu" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/versionoverride" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" @@ -83,6 +84,7 @@ type DynamoGraphDeploymentRunV1beta2Reconciler struct { Config *configv1alpha1.OperatorConfiguration RuntimeConfig *commoncontroller.RuntimeConfig Hardware dgdrRunHardwareDiscoverer + DynamoVersion string OperatorImage string OperatorImagePullPolicy corev1.PullPolicy } @@ -289,7 +291,7 @@ func (r *DynamoGraphDeploymentRunV1beta2Reconciler) reconcileInput( } } } - contents, snapshotPath, err := dgdrRunInput(run, input, execution.GPUsPerNode) + contents, snapshotPath, err := dgdrRunInput(run, input, r.DynamoVersion, execution.GPUsPerNode) if err != nil { return dgdrRunExecution{}, err } @@ -342,12 +344,13 @@ func (r *DynamoGraphDeploymentRunV1beta2Reconciler) discoverGPUsPerNode( func dgdrRunInput( run *nvidiacomv1beta2.DynamoGraphDeploymentRun, parameters map[string]any, + dynamoVersion string, gpusPerNode int, ) ([]byte, string, error) { if run.Spec.Recommendation == nil || run.Spec.Recommendation.MaxCandidates < 1 { return nil, "", errors.New("spec.recommendation.maxCandidates must be defaulted before the Run is created") } - underlay := map[string]any{"dgdr_run": map[string]any{ + output := map[string]any{ "name": run.Name, "snapshot_dir": dgdrRunSnapshotPath, "namespace": run.Namespace, @@ -355,7 +358,14 @@ func dgdrRunInput( "derive_runtime_images": true, "num_gpus_per_node": int64(gpusPerNode), "max_candidates": int64(run.Spec.Recommendation.MaxCandidates), - }} + } + if _, err := versionoverride.ParseImageVersion(run.Spec.AISimulate.Image); err != nil { + if _, versionErr := versionoverride.Parse(dynamoVersion); versionErr != nil { + return nil, "", fmt.Errorf("resolve Dynamo version for planner image %q: %w", run.Spec.AISimulate.Image, versionErr) + } + output["runtime_version_override"] = dynamoVersion + } + underlay := map[string]any{"dgdr_run": output} input := aisimulate.MergeParameters(underlay, parameters) snapshotPath := dgdrRunSnapshotPath if configured, found, err := unstructured.NestedString(input, "dgdr_run", "snapshot_dir"); err == nil && found && path.IsAbs(configured) { diff --git a/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller_test.go b/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller_test.go index f53949022e53..1cfc3dc4b6f4 100644 --- a/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller_test.go +++ b/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller_test.go @@ -73,6 +73,7 @@ func newDGDRRunFixture(t *testing.T, hardware dgdrRunHardwareDiscoverer, objects reconciler: &DynamoGraphDeploymentRunV1beta2Reconciler{ Client: c, Hardware: hardware, + DynamoVersion: "1.2.3", OperatorImage: "example.com/dynamo-operator:1.2.3", OperatorImagePullPolicy: corev1.PullAlways, }, @@ -172,6 +173,15 @@ func assertDGDRRunJobIsolation( if sweeper.Image != run.Spec.AISimulate.Image { t.Fatalf("sweeper image = %q, want %q", sweeper.Image, run.Spec.AISimulate.Image) } + if want := []string{"aisimulate"}; !reflect.DeepEqual(sweeper.Command, want) { + t.Fatalf("sweeper command = %q, want %q", sweeper.Command, want) + } + if want := []string{ + "recommend", "--stack", "dynamo", "--config", dgdrRunInputPath, + "--output", "dgdr_run", "--output-dir", "/work/results", + }; !reflect.DeepEqual(sweeper.Args, want) { + t.Fatalf("sweeper args = %q, want %q", sweeper.Args, want) + } if publisher.Image != operatorImage || publisher.ImagePullPolicy != corev1.PullAlways { t.Fatalf("publisher image = %q/%s, want configured operator image/Always", publisher.Image, publisher.ImagePullPolicy) } @@ -218,6 +228,46 @@ func assertDGDRRunInput(t *testing.T, c client.Client, job *batchv1.Job, hardwar } } +func TestDGDRRunInputResolvesRuntimeVersion(t *testing.T) { + tests := []struct { + name string + plannerImage string + dynamoVersion string + wantOverride string + wantErr bool + }{ + {name: "canonical planner tag", plannerImage: "example.com/dynamo-planner:1.7.0"}, + {name: "opaque CI tag", plannerImage: "example.com/dynamo-planner:git-sha", dynamoVersion: "1.7.0", wantOverride: "1.7.0"}, + {name: "opaque tag without canonical version", plannerImage: "example.com/dynamo-planner:git-sha", dynamoVersion: "1.7", wantErr: true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + run := dgdrRunForTest(t, map[string]any{}) + run.Spec.AISimulate.Image = tt.plannerImage + + contents, _, err := dgdrRunInput(run, map[string]any{}, tt.dynamoVersion, 8) + if (err != nil) != tt.wantErr { + t.Fatalf("lower input error = %v, wantErr %t", err, tt.wantErr) + } + if tt.wantErr { + return + } + input := map[string]any{} + if err := yaml.Unmarshal(contents, &input); err != nil { + t.Fatalf("decode input: %v", err) + } + output := input["dgdr_run"].(map[string]any) + got, found := output["runtime_version_override"] + if tt.wantOverride == "" && found { + t.Fatalf("dgdr_run.runtime_version_override = %v, want omitted", got) + } + if tt.wantOverride != "" && got != tt.wantOverride { + t.Fatalf("dgdr_run.runtime_version_override = %v, want %s", got, tt.wantOverride) + } + }) + } +} + func assertDGDRRunSupportResources( t *testing.T, c client.Client, diff --git a/deploy/operator/internal/controller/setup.go b/deploy/operator/internal/controller/setup.go index 94e96f1ac4bb..6fd8595c2f6e 100644 --- a/deploy/operator/internal/controller/setup.go +++ b/deploy/operator/internal/controller/setup.go @@ -49,6 +49,7 @@ type DynamoGraphDeploymentRunSetupOptions struct { SetupOptions GPUDiscoveryCache *gpu.GPUDiscoveryCache GPUDiscovery *gpu.GPUDiscovery + DynamoVersion string OperatorImage string OperatorImagePullPolicy corev1.PullPolicy } @@ -187,6 +188,7 @@ func SetupDynamoGraphDeploymentRunV1beta2(mgr ctrl.Manager, opts DynamoGraphDepl return opts.RuntimeConfig != nil && opts.RuntimeConfig.Gate.Enabled(features.GPUDiscovery) }, }, + DynamoVersion: opts.DynamoVersion, OperatorImage: opts.OperatorImage, OperatorImagePullPolicy: opts.OperatorImagePullPolicy, }).SetupWithManager(mgr); err != nil {