diff --git a/.github/codeowners/areas.yaml b/.github/codeowners/areas.yaml index 57f8b743f723..6035f83698be 100644 --- a/.github/codeowners/areas.yaml +++ b/.github/codeowners/areas.yaml @@ -888,6 +888,10 @@ shared: owners: - planner - operator +- glob: components/src/dynamo/aisimulate/output/dgdr_run/ + owners: + - planner + - operator - glob: components/src/dynamo/planner/connectors/kubernetes.py owners: - planner diff --git a/CODEOWNERS b/CODEOWNERS index 17aec522c0ad..6c3ca1f9580d 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -764,6 +764,7 @@ requirements.txt /deploy/operator/internal/webhook/**/*checkpoint* @ai-dynamo/dynamo-operator-codeowners @ai-dynamo/dynamo-gms-codeowners /.github/actions/build-deploy-component/action.yml @ai-dynamo/dynamo-ops-codeowners @ai-dynamo/dynamo-operator-codeowners @ai-dynamo/dynamo-gms-codeowners @ai-dynamo/dynamo-planner-codeowners /.github/actions/check-deploy-component/action.yml @ai-dynamo/dynamo-ops-codeowners @ai-dynamo/dynamo-operator-codeowners @ai-dynamo/dynamo-gms-codeowners +/components/src/dynamo/aisimulate/output/dgdr_run/ @ai-dynamo/dynamo-planner-codeowners @ai-dynamo/dynamo-operator-codeowners /components/src/dynamo/common/backend/publisher.py @ai-dynamo/dynamo-runtime-codeowners @ai-dynamo/dynamo-kv-memory-codeowners /components/src/dynamo/common/backend/telemetry.py @ai-dynamo/dynamo-runtime-codeowners @ai-dynamo/dynamo-observability-codeowners /components/src/dynamo/common/utils/image_utils.py @ai-dynamo/dynamo-runtime-codeowners @ai-dynamo/dynamo-diffusion-codeowners diff --git a/components/src/dynamo/aisimulate/output/dgdr_run/__init__.py b/components/src/dynamo/aisimulate/output/dgdr_run/__init__.py new file mode 100644 index 000000000000..290ce406d8f5 --- /dev/null +++ b/components/src/dynamo/aisimulate/output/dgdr_run/__init__.py @@ -0,0 +1,34 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Live DynamoGraphDeploymentRun output adapter for AISimulate (``dgdr_run``).""" + +from dynamo.aisimulate.output.dgdr_run.adapter import ( + DGDRRunOutputAdapter, + DGDRRunOutputConfig, + DGDRRunOutputPlugin, + candidate_id_for, + create_adapter, +) +from dynamo.aisimulate.output.dgdr_run.snapshot import ( + SNAPSHOT_FILE_NAME, + CandidateOutcome, + DGDRRunSnapshot, + RunPhase, + SnapshotCandidate, + write_snapshot, +) + +__all__ = [ + "SNAPSHOT_FILE_NAME", + "CandidateOutcome", + "DGDRRunOutputAdapter", + "DGDRRunOutputConfig", + "DGDRRunOutputPlugin", + "DGDRRunSnapshot", + "RunPhase", + "SnapshotCandidate", + "candidate_id_for", + "create_adapter", + "write_snapshot", +] diff --git a/components/src/dynamo/aisimulate/output/dgdr_run/adapter.py b/components/src/dynamo/aisimulate/output/dgdr_run/adapter.py new file mode 100644 index 000000000000..72869b16cacd --- /dev/null +++ b/components/src/dynamo/aisimulate/output/dgdr_run/adapter.py @@ -0,0 +1,696 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Live DGDRRun output adapter (``--output dgdr_run``). + +Independent of the standalone ``dgd`` adapter, which is called once with the +final recommendation. This adapter subscribes to Sweeper's ``on_round`` and +``on_candidate`` callbacks for the whole search, keeps the *current bounded +projection* of the run in memory, and periodically publishes it as an +atomically replaced ``dgdr_run_snapshot.yaml`` (see ``snapshot.py``) that the +publisher sidecar in the same pod consumes. + +Both adapters share one Candidate-to-DGD implementation through +``dynamo.aisimulate.output.dgd.renderers`` (its public +CandidateMaterializationError / DGDGenerationOptions / render_dgd surface); +nothing else is shared. + +Design points (all adapter-specific, none of them shared with ``dgd``): + +* **Callbacks never block and never lose state.** They only update the latest + in-memory projection under a short lock and wake the writer. Nothing is + queued, so nothing can be dropped under pressure: only *file writes* are + coalesced, never logical state changes. +* **Bounded.** At most ``max_candidates`` points are retained and published, + no matter how many are evaluated. The bound comes from the run's + ``recommendation.maxCandidates``; it is not derived from any transport. +* **Rate limited.** The writer publishes the latest complete state at most once + per ``snapshot_interval_seconds``. The initial snapshot (``start()``) and the + terminal snapshot (``close()``) bypass the limit. +* **Materialize once.** A point's DGD is rendered the first time it enters the + projection and cached by its stable id, so progress or ordering changes never + re-render retained points. +* **Terminal snapshot is a completion barrier.** ``close()`` stops accepting + updates, folds in everything accepted so far, and synchronously writes the + terminal snapshot before returning. A failure to write it is raised, never + swallowed, so the Sweeper container cannot exit successfully without it. + +Scalar searches only for now (best-first ordering by score); a Pareto +selection policy is a separate piece of work and is rejected explicitly. +""" + +from __future__ import annotations + +import hashlib +import json +import logging +import math +import threading +import time +from collections.abc import Mapping, Sequence +from dataclasses import dataclass, fields +from pathlib import Path +from typing import Any, Literal, Protocol + +import yaml +from aisimulate.output_adapter import ( + OUTPUT_ADAPTER_API_VERSION, + RecommendationOutputCallbacks, +) +from aisimulate.sweeper.config import SmartSearchConfig +from aisimulate.sweeper.result import SweepResult +from pydantic import BaseModel, ConfigDict, Field + +from dynamo.aisimulate.output.dgd.renderers import ( + CandidateMaterializationError, + DGDGenerationOptions, + render_dgd, +) +from dynamo.aisimulate.output.dgdr_run.snapshot import ( + SNAPSHOT_FILE_NAME, + CandidateOutcome, + DGDRRunSnapshot, + RunPhase, + SnapshotCandidate, + write_snapshot, +) +from dynamo.common.container_images import derive_backend_image + +logger = logging.getLogger(__name__) + +_ID_PREFIX = "evaluated-point-" +_MAX_RENDER_ATTEMPTS = 3 +_MAX_ERROR_MESSAGE = 500 +_WRITE_RETRY_BACKOFF = 1.0 +_ID_HEX_LENGTH = 12 + + +class CandidateRecordLike(Protocol): + """The subset of aisimulate's ``CandidateRecord`` this adapter reads.""" + + candidate_id: str + status: Any # aisimulate.sweeper.result.CandidateStatus + score: float | None + used_gpus: int | None + config: dict[str, Any] + + +def _plain(value: Any) -> Any: + """Copy ``value`` into plain, YAML-safe containers (detached from the caller).""" + return json.loads(json.dumps(value, default=str, sort_keys=True)) + + +def _bounded(text: str) -> str: + return text[:_MAX_ERROR_MESSAGE] + + +def _finite(value: Any) -> Any: + """Replace non-finite floats with ``None`` so the snapshot stays valid JSON/YAML.""" + if isinstance(value, float): + return value if math.isfinite(value) else None + if isinstance(value, dict): + return {key: _finite(item) for key, item in value.items()} + if isinstance(value, list): + return [_finite(item) for item in value] + return value + + +class _RendererUnavailable(Exception): + """Rendering failed for a reason that is not specific to one candidate.""" + + +def candidate_id_for(parameters: Mapping[str, Any]) -> str: + """Stable identity of an evaluated point. + + Depends only on the point's resolved parameters: never on rank, list + position, or the order in which points were observed. + """ + canonical = json.dumps(parameters, sort_keys=True, separators=(",", ":")) + digest = hashlib.sha256(canonical.encode("utf-8")).hexdigest() + return f"{_ID_PREFIX}{digest[:_ID_HEX_LENGTH]}" + + +def _is_feasible(record: CandidateRecordLike) -> bool: + # By value, not identity: callers may pass aisimulate's CandidateStatus + # enum, a plain string, or a test double. + return str(getattr(record.status, "value", record.status)) == "feasible" + + +@dataclass(frozen=True) +class _Retained: + """One retained point: immutable evaluation facts, no rendering.""" + + id: str + score: float + used_gpus: int + parameters: dict[str, Any] + metrics: dict[str, Any] + + @property + def rank_key(self) -> tuple[float, int, str]: + """Ascending sort key: best first. + + Mirrors Sweeper's scalar ``rank()``: highest score, then fewer GPUs, + then the canonical config JSON. It is only used to bound the live + retained set; the final snapshot keeps Sweeper's own selection order. + """ + return ( + -self.score, + self.used_gpus, + json.dumps(self.parameters, sort_keys=True, separators=(",", ":")), + ) + + +@dataclass(frozen=True) +class _Terminal: + phase: RunPhase + message: str + error: str + + +class DGDRRunOutputConfig(BaseModel): + """Configuration accepted for ``--output dgdr_run``.""" + + model_config = ConfigDict(extra="forbid") + + name: str = Field(min_length=1, pattern=r"^[a-z0-9]([-a-z0-9]*[a-z0-9])?$") + snapshot_dir: str | None = Field(default=None, min_length=1) + namespace: str | None = None + generator: Literal["aic", "direct"] = "aic" + runtime_image: str = Field(min_length=1) + # A Run searches multiple backends from one planner image. Standalone callers keep + # the historical fixed-image behavior unless they opt into per-candidate derivation. + derive_runtime_images: bool = False + runtime_version_override: str | None = None + num_gpus_per_node: int = Field(gt=0) + max_candidates: int = Field(default=5, ge=1) + snapshot_interval_seconds: float = Field(default=5.0, ge=0) + + def generation_options(self, backend: str) -> DGDGenerationOptions: + runtime_image = ( + derive_backend_image(self.runtime_image, backend) + if self.derive_runtime_images + else self.runtime_image + ) + return DGDGenerationOptions( + runtime_image=runtime_image, + runtime_version_override=self.runtime_version_override, + namespace=self.namespace, + num_gpus_per_node=self.num_gpus_per_node, + ) + + +class DGDRRunOutputAdapter: + """Publishes the current bounded projection of one Sweeper run. + + One instance per run. Pass the bound methods to ``Sweeper.run`` and use the + adapter as a context manager (or call ``start()``/``close()``) so the + terminal snapshot is always written, including when the run raises:: + + with DGDRRunOutputAdapter(config, workload, snapshot_dir=path) as adapter: + sweeper.run(on_round=adapter.on_round, on_candidate=adapter.on_candidate) + """ + + def __init__( + self, + config: DGDRRunOutputConfig, + workload: Any, + *, + snapshot_dir: Path | None = None, + is_pareto: bool = False, + keep_arrival_order: bool = False, + ) -> None: + if is_pareto: + raise NotImplementedError( + "dgdr_run only supports scalar objectives so far; Pareto " + "retention needs its own selection policy rather than an " + "approximation of top-N-by-score" + ) + directory = snapshot_dir if snapshot_dir is not None else config.snapshot_dir + if directory is None: + raise ValueError("a snapshot directory is required") + self._directory = Path(directory) + self._config = config + # Validate the source image and runtime version before the first callback. + config.generation_options("vllm") + self._workload = workload + self._max_candidates = config.max_candidates + self._interval = config.snapshot_interval_seconds + + # Latest in-memory desired state. Callbacks mutate it under _lock. + self._lock = threading.Lock() + self._retained: dict[str, _Retained] = {} + self._round_no = 0 + self._evaluated = 0 + self._terminal: _Terminal | None = None + self._accepting = True + + # Writer coordination. + self._wake = threading.Event() + self._stop = threading.Event() + self._thread: threading.Thread | None = None + self._next_allowed = float("-inf") + + # Materialization cache, keyed by stable id. Touched only while + # holding _publish_lock. A failure is retried on later writes up to + # _MAX_RENDER_ATTEMPTS in total, because a first render can fail + # transiently (cold start); after that it is final. + # The final result is already ranked by Sweeper (goal and SLA aware); + # publishing it must not re-rank it. + self._keep_arrival_order = keep_arrival_order + self._publish_lock = threading.Lock() + self._materialized: dict[str, SnapshotCandidate] = {} + self._render_attempts: dict[str, int] = {} + self._writes = 0 + + # -- observability --------------------------------------------------- + + @property + def snapshot_writes(self) -> int: + """Number of snapshots written so far (initial and terminal included).""" + return self._writes + + @property + def snapshot_path(self) -> Path: + return self._directory / SNAPSHOT_FILE_NAME + + # -- Sweeper callbacks: update latest state, never block, never raise -- + + def on_candidate(self, record: CandidateRecordLike) -> None: + try: + retained = self._retain(record) if _is_feasible(record) else None + except Exception: # a callback must not break the search + logger.exception("dgdr_run: ignoring candidate that could not be read") + retained = None + with self._lock: + if not self._accepting: + return + self._evaluated += 1 + # First observation wins: an evaluated point is immutable, so a + # re-evaluation of the same point never changes what was published. + if retained is not None and retained.id not in self._retained: + self._retained[retained.id] = retained + if len(self._retained) > self._max_candidates: + worst = max(self._retained.values(), key=lambda r: r.rank_key) + del self._retained[worst.id] + self._wake.set() + + def on_round(self, round_number: int, candidates: list[Any]) -> None: + del candidates # cumulative feasible list: the adapter tracks its own set + with self._lock: + if not self._accepting: + return + self._round_no = max(self._round_no, round_number) + self._wake.set() + + def report_progress(self, *, evaluated: int, round_no: int = 0) -> None: + """Raise the evaluated count and round to authoritative values (never lowers them).""" + with self._lock: + if not self._accepting: + return + self._evaluated = max(self._evaluated, evaluated) + self._round_no = max(self._round_no, round_no) + self._wake.set() + + @staticmethod + def _retain(record: CandidateRecordLike) -> _Retained | None: + if record.score is None or record.used_gpus is None: + logger.warning( + "dgdr_run: feasible candidate %s has no score/used_gpus; skipped", + record.candidate_id, + ) + return None + parameters = _plain(dict(record.config)) + score = float(record.score) + metrics: dict[str, Any] = { + "score": score if math.isfinite(score) else None, + "usedGpus": int(record.used_gpus), + } + objectives = getattr(record, "objectives", None) + if objectives: + metrics["objectives"] = _finite(_plain(dict(objectives))) + return _Retained( + id=candidate_id_for(parameters), + # A NaN score has no place in an ordering; it ranks as the worst. + score=-math.inf if math.isnan(score) else score, + used_gpus=int(record.used_gpus), + parameters=parameters, + metrics=metrics, + ) + + # -- lifecycle --------------------------------------------------------- + + def start(self) -> None: + """Write the initial snapshot immediately, then start the writer.""" + if self._thread is not None: + raise RuntimeError("DGDRRunOutputAdapter already started") + self._publish() + self._next_allowed = time.monotonic() + self._interval + self._thread = threading.Thread( + target=self._writer_loop, name="dgdr-run-snapshot-writer", daemon=True + ) + self._thread.start() + + def close( + self, + *, + phase: RunPhase = RunPhase.SUCCEEDED, + message: str = "", + error: str = "", + ) -> None: + """Write the terminal snapshot; returns only once it is on disk. + + Stops accepting updates, folds in every update accepted before this + call, bypasses the rate limit, and replaces the file synchronously. + Raises if the terminal snapshot cannot be written. Safe to call without + ``start()`` and safe to call twice (the second call is a no-op). + """ + if not phase.terminal: + raise ValueError("close() requires a terminal phase") + with self._lock: + if self._terminal is not None: + return + self._accepting = False + self._terminal = _Terminal(phase=phase, message=message, error=error) + self._stop.set() + self._wake.set() + if self._thread is not None: + self._thread.join() + self._publish_or_record_failure() + + def _publish_or_record_failure(self) -> None: + """Publish; if rendering itself is broken, record that as a terminal failure. + + A broken renderer is a run-level problem, not a candidate-specific one, so it + must not become ``MATERIALIZATION_FAILED``. The terminal ``Failed`` snapshot + carries the candidates that were already rendered, and the original exception + is re-raised. Failures to write the file keep propagating unchanged. + """ + try: + self._publish() + except _RendererUnavailable as unavailable: + cause = unavailable.__cause__ or unavailable + logger.error("dgdr_run: renderer unavailable", exc_info=cause) + with self._lock: + self._accepting = False + self._terminal = _Terminal( + phase=RunPhase.FAILED, + message=_bounded(str(cause)), + error=type(cause).__name__, + ) + self._publish(render=False) + raise cause + + def finish_search(self) -> None: + """The search ended normally: flush the last live state and stop the writer. + + Not a terminal state by itself: the terminal ``Succeeded`` snapshot is written + afterwards by ``write()``, which sees the final result. A search that found no + feasible candidate never reaches ``write()``, so that case is recorded here as + ``Failed`` instead of being left looking in progress. + """ + with self._lock: + if self._terminal is not None: + return + nothing_found = not self._retained + if nothing_found: + self.close(phase=RunPhase.FAILED, message="no feasible candidate found") + return + self._stop.set() + self._wake.set() + if self._thread is not None: + self._thread.join() + self._publish_or_record_failure() + + def fail(self, error: BaseException) -> None: + """The search raised: write the terminal ``Failed`` snapshot.""" + self.close( + phase=RunPhase.FAILED, + error=type(error).__name__, + message=_bounded(str(error)), + ) + + def __enter__(self) -> DGDRRunOutputAdapter: + self.start() + return self + + def __exit__(self, exc_type: type[BaseException] | None, *_: Any) -> None: + if exc_type is None: + self.close(phase=RunPhase.SUCCEEDED) + return + try: + self.close(phase=RunPhase.FAILED, error=exc_type.__name__) + except Exception: # never mask the run's own exception + logger.exception("dgdr_run: could not write the terminal snapshot") + + # -- writer ---------------------------------------------------------- + + def _writer_loop(self) -> None: + while True: + self._wake.wait() + if self._stop.is_set(): + return + delay = self._next_allowed - time.monotonic() + if delay > 0 and self._stop.wait(delay): + return + wait = self._interval + try: + self._publish() + except Exception: # keep the writer alive; retry + logger.exception("dgdr_run: snapshot write failed; will retry") + self._wake.set() + # Even with a zero interval, a persistent failure must not spin. + wait = max(wait, _WRITE_RETRY_BACKOFF) + self._next_allowed = time.monotonic() + wait + + def _publish(self, *, render: bool = True) -> None: + with self._publish_lock: + with self._lock: + # Cleared together with the copy so no update is ever lost: an + # update after this point sets the event again. + self._wake.clear() + ordered = ( + list(self._retained.values()) + if self._keep_arrival_order + else sorted(self._retained.values(), key=lambda r: r.rank_key) + ) + round_no = self._round_no + evaluated = self._evaluated + terminal = self._terminal + if render: + candidates = tuple(self._materialize(item) for item in ordered) + else: + candidates = tuple( + self._materialized[item.id] + for item in ordered + if item.id in self._materialized + ) + live_ids = {item.id for item in ordered} + for stale in set(self._materialized) - live_ids: + del self._materialized[stale] + self._render_attempts.pop(stale, None) + write_snapshot( + self._directory, + DGDRRunSnapshot( + phase=terminal.phase if terminal else RunPhase.RUNNING, + round_no=round_no, + evaluated=evaluated, + candidates=candidates, + message=terminal.message if terminal else "", + error=terminal.error if terminal else "", + ), + ) + self._writes += 1 + + def _materialize(self, item: _Retained) -> SnapshotCandidate: + cached = self._materialized.get(item.id) + if cached is not None and ( + cached.outcome is CandidateOutcome.MATERIALIZED + or self._render_attempts.get(item.id, 0) >= _MAX_RENDER_ATTEMPTS + ): + return cached + self._render_attempts[item.id] = self._render_attempts.get(item.id, 0) + 1 + try: + manifest = render_dgd( + _RenderInput(config=item.parameters, used_gpus=item.used_gpus), + self._workload, + self._config.generation_options( + str(item.parameters.get("backend", "")) + ), + dgd_name=f"{self._config.name}-{item.id.removeprefix(_ID_PREFIX)}", + renderer=self._config.generator, + ) + entry = SnapshotCandidate( + id=item.id, + outcome=CandidateOutcome.MATERIALIZED, + parameters=item.parameters, + metrics=item.metrics, + manifest=manifest, + ) + except CandidateMaterializationError as exc: + entry = SnapshotCandidate( + id=item.id, + outcome=CandidateOutcome.MATERIALIZATION_FAILED, + parameters=item.parameters, + metrics=item.metrics, + error=_bounded(str(exc)), + ) + except Exception as exc: + raise _RendererUnavailable(str(exc)) from exc + self._materialized[item.id] = entry + return entry + + +@dataclass(frozen=True) +class _RenderInput: + """The evaluated candidate as the renderer sees it: its config and GPU count.""" + + config: dict[str, Any] + used_gpus: int + + +class DGDRRunOutputPlugin: + """The object AISimulate discovers for ``--output dgdr_run``. + + AISimulate calls ``subscribe(config, context=...)`` once, in the process that + runs the search, and wires the returned callbacks into the Sweeper: that is + the live path (initial snapshot immediately, then coalesced snapshots). + ``write()`` runs afterwards in the CLI process, only when the search + succeeded with a selection, and publishes the authoritative terminal + snapshot from the final result. A search that fails or finds nothing never + reaches ``write()``, so it leaves no terminal snapshot; the publisher + detects that from the Sweeper container's exit. + """ + + name = "dgdr_run" + api_version = OUTPUT_ADAPTER_API_VERSION + + def subscribe( + self, config: Mapping[str, Any], context: Any = None + ) -> RecommendationOutputCallbacks | None: + """Validate the configuration and, when possible, start live publishing. + + ``context`` (``RecommendationOutputContext``) carries the Sweeper workload, + which rendering needs. Without a context or without ``snapshot_dir`` there + is nothing to publish live, so only the config is validated and the final + snapshot is still produced by ``write()``. + """ + resolved = DGDRRunOutputConfig.model_validate(dict(config)) + resolved.generation_options("vllm") + workload = getattr(context, "workload", None) + if workload is None or resolved.snapshot_dir is None: + return None + # Live publishing needs both lifecycle hooks: without on_complete nothing stops + # the live writer before write() publishes the terminal snapshot, which a pending + # live write could then overwrite. Older AISimulate releases lack them; the + # terminal snapshot then comes from write() alone. + supported = {field.name for field in fields(RecommendationOutputCallbacks)} + if not {"on_complete", "on_failure"} <= supported: + logger.info( + "dgdr_run: AISimulate has no lifecycle callbacks; not publishing live" + ) + return None + adapter = DGDRRunOutputAdapter(resolved, workload) + adapter.start() + return RecommendationOutputCallbacks( + on_candidate=adapter.on_candidate, + on_round=adapter.on_round, + on_complete=adapter.finish_search, + on_failure=adapter.fail, + ) + + def live( + self, + config: Mapping[str, Any], + *, + workload: Any, + is_pareto: bool = False, + ) -> DGDRRunOutputAdapter: + resolved = DGDRRunOutputConfig.model_validate(dict(config)) + return DGDRRunOutputAdapter(resolved, workload, is_pareto=is_pareto) + + def write( + self, + config: Mapping[str, Any], + *, + result: SweepResult, + output_dir: Path, + ) -> Sequence[str | Path]: + resolved = DGDRRunOutputConfig.model_validate(dict(config)) + if result.views.pareto_front: + raise NotImplementedError("dgdr_run does not support Pareto results yet") + selected = list(result.selected_candidates)[: resolved.max_candidates] + if not selected: + raise CandidateMaterializationError("no feasible candidate found") + workload = SmartSearchConfig.model_validate(result.provenance.config).workload + directory = Path(resolved.snapshot_dir or output_dir) + adapter = DGDRRunOutputAdapter( + resolved, workload, snapshot_dir=directory, keep_arrival_order=True + ) + for candidate in selected: + adapter.on_candidate(_as_feasible(candidate)) + counts = getattr(result, "counts", None) + evaluated = getattr(counts, "evaluated", None) + adapter.report_progress( + evaluated=evaluated if isinstance(evaluated, int) else 0, + # The final result does not carry the round, but the live snapshot + # written by the search process does; progress never goes backwards. + round_no=_last_published_round(directory), + ) + adapter.close(phase=RunPhase.SUCCEEDED) + # AISimulate requires reported artifacts to live under output_dir; a snapshot + # on the shared volume does not, and is consumed by the publisher instead. + if directory.resolve() == Path(output_dir).resolve(): + return [Path(SNAPSHOT_FILE_NAME)] + return [] + + +def _last_published_round(directory: Path) -> int: + """The round recorded by the live snapshot already in ``directory``, else 0.""" + try: + loaded = yaml.safe_load((directory / SNAPSHOT_FILE_NAME).read_text()) + round_no = loaded["progress"]["round"] + except (OSError, yaml.YAMLError, KeyError, TypeError): + return 0 + return round_no if isinstance(round_no, int) and round_no > 0 else 0 + + +@dataclass(frozen=True) +class _FinalCandidate: + candidate_id: str + config: dict[str, Any] + score: float | None + used_gpus: int | None + objectives: dict[str, float] | None = None + status: str = "feasible" + + +def _as_feasible(candidate: Any) -> _FinalCandidate: + score = getattr(candidate, "score", None) + used_gpus = getattr(candidate, "used_gpus", None) + if score is None or used_gpus is None: + raise CandidateMaterializationError( + "selected candidate has no score/used_gpus, so it cannot be ranked" + ) + return _FinalCandidate( + candidate_id=str(getattr(candidate, "candidate_id", "")), + config=dict(candidate.config), + score=score, + used_gpus=used_gpus, + objectives=getattr(candidate, "objectives", None), + ) + + +def create_adapter() -> DGDRRunOutputPlugin: + """Create the Dynamo plugin discovered by AISimulate.""" + return DGDRRunOutputPlugin() + + +__all__ = [ + "CandidateRecordLike", + "DGDRRunOutputAdapter", + "DGDRRunOutputConfig", + "DGDRRunOutputPlugin", + "candidate_id_for", + "create_adapter", +] diff --git a/components/src/dynamo/aisimulate/output/dgdr_run/snapshot.py b/components/src/dynamo/aisimulate/output/dgdr_run/snapshot.py new file mode 100644 index 000000000000..f14ff16264ed --- /dev/null +++ b/components/src/dynamo/aisimulate/output/dgdr_run/snapshot.py @@ -0,0 +1,148 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Desired-state snapshot of one DGDRRun, written to the pod-local volume. + +The Sweeper container atomically replaces a single YAML file with the complete +*current* projection of one ``DynamoGraphDeploymentRun``; the publisher +sidecar in the same pod consumes it directly (shared ``emptyDir``). Nothing +here involves a ConfigMap, the Dynamo event plane, or any cross-pod RPC. + +The snapshot is desired state, not a command list and not history: + +* ``run`` carries progress and, once ``terminal`` is true, the final outcome; +* ``candidates`` is the bounded set of evaluated points that should exist as + ``DynamoGraphDeploymentCandidate`` objects, best-first for scalar searches. + +Each candidate has a stable ``id`` that depends only on the evaluated point +(its resolved parameters), never on its list position or rank. Reordering +the list must therefore never change an identity or a rendered manifest; it +only changes the ordered references the publisher writes to the run status. + +Each published snapshot is complete. The publisher only needs the latest one, +so intermediate states may be coalesced away, but a terminal snapshot must +always be written (see ``DGDRRunOutputAdapter.close``). +""" + +from __future__ import annotations + +import time +from dataclasses import dataclass, field +from enum import Enum +from pathlib import Path +from typing import Any + +import yaml + +from dynamo.aisimulate.output.dgd.writers.atomic import replace_text + +SNAPSHOT_FILE_NAME = "dgdr_run_snapshot.yaml" +SCHEMA_VERSION = 1 + + +class RunPhase(str, Enum): + """Run lifecycle as reported by the producer. Anything but RUNNING is terminal.""" + + RUNNING = "Running" + SUCCEEDED = "Succeeded" + FAILED = "Failed" + + @property + def terminal(self) -> bool: + return self is not RunPhase.RUNNING + + +class CandidateOutcome(str, Enum): + MATERIALIZED = "materialized" + MATERIALIZATION_FAILED = "materialization_failed" + + +@dataclass(frozen=True) +class SnapshotCandidate: + """One evaluated point in the bounded projection. + + ``parameters`` and ``metrics`` are the immutable evaluation facts the + publisher needs to build the candidate's resource: a DGD manifest alone + does not identify an evaluated point, because two evaluations can render + the same DGD under different parameters and still have different metrics. + """ + + id: str + outcome: CandidateOutcome + parameters: dict[str, Any] = field(default_factory=dict) + metrics: dict[str, Any] = field(default_factory=dict) + manifest: str | None = None + error: str | None = None + + def __post_init__(self) -> None: + if not self.id: + raise ValueError("candidate id must not be empty") + if self.outcome is CandidateOutcome.MATERIALIZED: + if not self.manifest: + raise ValueError("a materialized candidate must carry its manifest") + if self.error is not None: + raise ValueError("a materialized candidate must not carry an error") + elif not self.error: + raise ValueError("a failed candidate must carry its error") + elif self.manifest is not None: + raise ValueError("a failed candidate must not carry a manifest") + + def to_dict(self) -> dict[str, Any]: + payload: dict[str, Any] = { + "id": self.id, + "outcome": self.outcome.value, + "parameters": self.parameters, + "metrics": self.metrics, + } + if self.manifest is not None: + payload["manifest"] = self.manifest + if self.error is not None: + payload["error"] = self.error + return payload + + +@dataclass(frozen=True) +class DGDRRunSnapshot: + """The complete current projection of one run.""" + + phase: RunPhase + round_no: int + evaluated: int + candidates: tuple[SnapshotCandidate, ...] = () + message: str = "" + error: str = "" + + def to_dict(self) -> dict[str, Any]: + run: dict[str, Any] = { + "phase": self.phase.value, + "terminal": self.phase.terminal, + } + if self.message: + run["message"] = self.message + if self.error: + run["error"] = self.error + return { + "schemaVersion": SCHEMA_VERSION, + "timestamp": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + "run": run, + "progress": {"round": self.round_no, "evaluated": self.evaluated}, + "candidates": [candidate.to_dict() for candidate in self.candidates], + } + + +def write_snapshot(directory: Path, snapshot: DGDRRunSnapshot) -> Path: + """Atomically replace the snapshot file; readers never see a partial file.""" + path = directory / SNAPSHOT_FILE_NAME + replace_text(path, yaml.safe_dump(snapshot.to_dict(), sort_keys=False)) + return path + + +__all__ = [ + "SCHEMA_VERSION", + "SNAPSHOT_FILE_NAME", + "CandidateOutcome", + "DGDRRunSnapshot", + "RunPhase", + "SnapshotCandidate", + "write_snapshot", +] diff --git a/components/src/dynamo/aisimulate/tests/dgdr_run/test_adapter.py b/components/src/dynamo/aisimulate/tests/dgdr_run/test_adapter.py new file mode 100644 index 000000000000..7dd3b78732ed --- /dev/null +++ b/components/src/dynamo/aisimulate/tests/dgdr_run/test_adapter.py @@ -0,0 +1,1011 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import threading +import time +from dataclasses import dataclass +from pathlib import Path +from types import SimpleNamespace +from typing import Any + +import pytest +import yaml + +from dynamo.aisimulate.output.dgd.renderers import CandidateMaterializationError +from dynamo.aisimulate.output.dgdr_run import adapter as adapter_module +from dynamo.aisimulate.output.dgdr_run.adapter import ( + DGDRRunOutputAdapter, + DGDRRunOutputConfig, + candidate_id_for, + create_adapter, +) +from dynamo.aisimulate.output.dgdr_run.snapshot import SNAPSHOT_FILE_NAME, RunPhase + +pytestmark = [ + pytest.mark.unit, + pytest.mark.gpu_0, + pytest.mark.pre_merge, + pytest.mark.planner, + pytest.mark.parallel, +] + + +@dataclass +class Record: + candidate_id: str + score: float | None + used_gpus: int | None + config: dict[str, Any] + status: str = "feasible" + objectives: dict[str, float] | None = None + + +def record(tag: str, score: float, gpus: int = 8, **kwargs: Any) -> Record: + return Record( + candidate_id=f"sweeper-{tag}", + score=score, + used_gpus=gpus, + config={"backend": "vllm", "backend_version": "0.1", "tag": tag}, + **kwargs, + ) + + +def make_config(tmp_path: Path, **overrides: Any) -> DGDRRunOutputConfig: + values: dict[str, Any] = { + "name": "sweep", + "snapshot_dir": str(tmp_path), + "runtime_image": "nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.2.3", + "num_gpus_per_node": 8, + "max_candidates": 3, + "snapshot_interval_seconds": 0, + } + values.update(overrides) + return DGDRRunOutputConfig(**values) + + +def load(tmp_path: Path) -> dict[str, Any]: + return yaml.safe_load((tmp_path / SNAPSHOT_FILE_NAME).read_text()) + + +def manifests_by_id(snapshot: dict[str, Any]) -> dict[str, str]: + return {c["id"]: c["manifest"] for c in snapshot["candidates"]} + + +def tags(snapshot: dict[str, Any]) -> list[str]: + return [candidate["parameters"]["tag"] for candidate in snapshot["candidates"]] + + +def wait_for(predicate: Any, timeout: float = 5.0) -> None: + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if predicate(): + return + time.sleep(0.01) + raise AssertionError("condition not reached before the timeout") + + +@pytest.fixture +def renders(monkeypatch: pytest.MonkeyPatch) -> list[str]: + """Replace the shared materializer with a counting fake.""" + calls: list[str] = [] + + def fake_render(candidate: Any, workload: Any, options: Any, **kwargs: Any) -> str: + tag = candidate.config["tag"] + calls.append(tag) + if tag.startswith("bad"): + raise CandidateMaterializationError(f"cannot render {tag}") + return ( + "apiVersion: nvidia.com/v1beta1\n" + "kind: DynamoGraphDeployment\n" + f"metadata:\n name: {kwargs['dgd_name']}\n" + f"spec:\n marker: {tag}\n" + ) + + monkeypatch.setattr(adapter_module, "render_dgd", fake_render) + return calls + + +# -- projection --------------------------------------------------------------- + + +def test_start_writes_the_initial_snapshot_immediately( + tmp_path: Path, renders: list[str] +) -> None: + config = make_config(tmp_path, snapshot_interval_seconds=60) + with DGDRRunOutputAdapter(config, workload=None): + snapshot = load(tmp_path) + assert snapshot["run"] == {"phase": "Running", "terminal": False} + assert snapshot["candidates"] == [] + assert snapshot["progress"] == {"round": 0, "evaluated": 0} + + +def test_projection_is_bounded_and_best_first_whatever_is_evaluated( + tmp_path: Path, renders: list[str] +) -> None: + adapter = DGDRRunOutputAdapter(make_config(tmp_path), workload=None) + for index in range(500): + adapter.on_candidate(record(f"p{index:03d}", score=float(index))) + adapter.close() + + snapshot = load(tmp_path) + assert tags(snapshot) == ["p499", "p498", "p497"] + assert snapshot["progress"]["evaluated"] == 500 + # Only retained points are ever materialized, never all 500. + assert len(renders) == 3 + + +def test_ties_break_on_fewer_gpus_then_id_deterministically( + tmp_path: Path, renders: list[str] +) -> None: + adapter = DGDRRunOutputAdapter(make_config(tmp_path, max_candidates=4), None) + adapter.on_candidate(record("big", score=5.0, gpus=16)) + adapter.on_candidate(record("small", score=5.0, gpus=8)) + adapter.on_candidate(record("best", score=9.0, gpus=64)) + adapter.close() + + assert tags(load(tmp_path)) == ["best", "small", "big"] + + +def test_infeasible_candidates_are_counted_but_never_retained( + tmp_path: Path, renders: list[str] +) -> None: + adapter = DGDRRunOutputAdapter(make_config(tmp_path), workload=None) + adapter.on_candidate(record("ok", 1.0)) + adapter.on_candidate(record("nope", 99.0, status="infeasible")) + adapter.close() + + snapshot = load(tmp_path) + assert tags(snapshot) == ["ok"] + assert snapshot["progress"]["evaluated"] == 2 + + +def test_snapshot_preserves_the_evaluated_point_not_just_its_manifest( + tmp_path: Path, renders: list[str] +) -> None: + adapter = DGDRRunOutputAdapter(make_config(tmp_path), workload=None) + adapter.on_candidate(record("a", 87.7, gpus=4, objectives={"goodput": 12.5})) + adapter.close() + + (candidate,) = load(tmp_path)["candidates"] + assert candidate["outcome"] == "materialized" + assert candidate["parameters"] == { + "backend": "vllm", + "backend_version": "0.1", + "tag": "a", + } + assert candidate["metrics"] == { + "score": 87.7, + "usedGpus": 4, + "objectives": {"goodput": 12.5}, + } + assert "marker: a" in candidate["manifest"] + + +def test_each_candidate_uses_the_runtime_image_for_its_backend( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + images: dict[str, str] = {} + + def capture_image( + candidate: Any, workload: Any, options: Any, **kwargs: Any + ) -> str: + del workload, kwargs + images[candidate.config["backend"]] = options.runtime_image + return "apiVersion: nvidia.com/v1beta1\nkind: DynamoGraphDeployment\n" + + monkeypatch.setattr(adapter_module, "render_dgd", capture_image) + adapter = DGDRRunOutputAdapter( + make_config( + tmp_path, + runtime_image="nvcr.io/nvidia/ai-dynamo/dynamo-planner:1.2.3", + derive_runtime_images=True, + ), + workload=None, + ) + for index, backend in enumerate(("vllm", "sglang", "trtllm")): + candidate = record(backend, float(index)) + candidate.config["backend"] = backend + adapter.on_candidate(candidate) + adapter.close() + + assert images == { + "vllm": "nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.2.3", + "sglang": "nvcr.io/nvidia/ai-dynamo/sglang-runtime:1.2.3", + "trtllm": "nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.2.3", + } + + +def test_explicit_runtime_image_stays_fixed_by_default( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + images: list[str] = [] + + def capture_image( + candidate: Any, workload: Any, options: Any, **kwargs: Any + ) -> str: + del candidate, workload, kwargs + images.append(options.runtime_image) + return "apiVersion: nvidia.com/v1beta1\nkind: DynamoGraphDeployment\n" + + monkeypatch.setattr(adapter_module, "render_dgd", capture_image) + adapter = DGDRRunOutputAdapter( + make_config(tmp_path, runtime_image="example.com/my-runtime:1.2.3"), + workload=None, + ) + candidate = record("sglang", 1.0) + candidate.config["backend"] = "sglang" + adapter.on_candidate(candidate) + adapter.close() + + assert images == ["example.com/my-runtime:1.2.3"] + + +def test_round_progress_is_monotonic(tmp_path: Path, renders: list[str]) -> None: + adapter = DGDRRunOutputAdapter(make_config(tmp_path), workload=None) + adapter.on_round(3, []) + adapter.on_round(2, []) + adapter.close() + assert load(tmp_path)["progress"]["round"] == 3 + + +# -- identity, immutability, materialization ---------------------------------- + + +def test_identity_and_manifests_do_not_depend_on_rank_or_observation_order( + tmp_path: Path, renders: list[str] +) -> None: + first = tmp_path / "first" + second = tmp_path / "second" + one = DGDRRunOutputAdapter(make_config(first), workload=None) + one.on_candidate(record("a", 1.0)) + one.on_candidate(record("b", 2.0)) + one.close() + two = DGDRRunOutputAdapter(make_config(second), workload=None) + two.on_candidate(record("b", 1.0)) + two.on_candidate(record("a", 2.0)) + two.close() + + one_snapshot, two_snapshot = load(first), load(second) + assert tags(one_snapshot) == ["b", "a"] + assert tags(two_snapshot) == ["a", "b"] + assert manifests_by_id(one_snapshot) == manifests_by_id(two_snapshot) + + +def test_identity_depends_only_on_the_resolved_parameters() -> None: + assert candidate_id_for({"a": 1, "b": 2}) == candidate_id_for({"b": 2, "a": 1}) + assert candidate_id_for({"a": 1}) != candidate_id_for({"a": 2}) + assert candidate_id_for({"a": 1}).startswith("evaluated-point-") + + +def test_a_new_better_point_reorders_but_does_not_rematerialize_retained_ones( + tmp_path: Path, renders: list[str] +) -> None: + adapter = DGDRRunOutputAdapter(make_config(tmp_path), workload=None) + adapter.start() + adapter.on_candidate(record("a", 1.0)) + adapter.on_candidate(record("b", 2.0)) + wait_for(lambda: tags(load(tmp_path)) == ["b", "a"]) + before = {c["id"]: c["manifest"] for c in load(tmp_path)["candidates"]} + assert sorted(renders) == ["a", "b"] + + adapter.on_candidate(record("c", 3.0)) + wait_for(lambda: tags(load(tmp_path)) == ["c", "b", "a"]) + adapter.close() + + after = {c["id"]: c["manifest"] for c in load(tmp_path)["candidates"]} + assert {key: after[key] for key in before} == before + assert sorted(renders) == ["a", "b", "c"] + + +def test_progress_updates_do_not_rematerialize( + tmp_path: Path, renders: list[str] +) -> None: + adapter = DGDRRunOutputAdapter(make_config(tmp_path), workload=None) + adapter.start() + adapter.on_candidate(record("a", 1.0)) + wait_for(lambda: tags(load(tmp_path)) == ["a"]) + for round_no in range(1, 20): + adapter.on_round(round_no, []) + wait_for(lambda: load(tmp_path)["progress"]["round"] == 19) + adapter.close() + assert renders == ["a"] + + +def test_a_point_is_immutable_once_observed(tmp_path: Path, renders: list[str]) -> None: + adapter = DGDRRunOutputAdapter(make_config(tmp_path), workload=None) + adapter.on_candidate(record("a", 1.0)) + adapter.on_candidate(record("a", 50.0)) # same parameters, noisy re-evaluation + adapter.close() + + (candidate,) = load(tmp_path)["candidates"] + assert candidate["metrics"]["score"] == 1.0 + assert load(tmp_path)["progress"]["evaluated"] == 2 + + +def test_a_persistent_materialization_failure_is_reported_and_retried_a_bounded_number_of_times( + tmp_path: Path, renders: list[str] +) -> None: + adapter = DGDRRunOutputAdapter(make_config(tmp_path), workload=None) + adapter.start() + adapter.on_candidate(record("bad1", 5.0)) + wait_for(lambda: len(load(tmp_path)["candidates"]) == 1) + for round_no in range(1, 10): + adapter.on_round(round_no, []) + wait_for(lambda: load(tmp_path)["progress"]["round"] == 9) + adapter.close() + + (candidate,) = load(tmp_path)["candidates"] + assert candidate["outcome"] == "materialization_failed" + assert "cannot render bad1" in candidate["error"] + assert "manifest" not in candidate + assert renders == ["bad1"] * adapter_module._MAX_RENDER_ATTEMPTS + + +# -- rate limiting and coalescing --------------------------------------------- + + +def test_many_callbacks_in_one_interval_produce_one_periodic_write( + tmp_path: Path, renders: list[str] +) -> None: + config = make_config(tmp_path, snapshot_interval_seconds=0.5) + adapter = DGDRRunOutputAdapter(config, workload=None) + adapter.start() + assert adapter.snapshot_writes == 1 + + for index in range(300): + adapter.on_candidate(record(f"p{index:03d}", score=float(index))) + time.sleep(0.15) + assert adapter.snapshot_writes == 1 + + wait_for(lambda: adapter.snapshot_writes == 2, timeout=5) + snapshot = load(tmp_path) + # The one periodic write carries the latest state, not an arbitrary + # intermediate one. + assert snapshot["progress"]["evaluated"] == 300 + assert tags(snapshot) == ["p299", "p298", "p297"] + + time.sleep(0.2) + assert adapter.snapshot_writes == 2 + adapter.close() + + +def test_terminal_state_bypasses_the_rate_limit_and_is_always_flushed( + tmp_path: Path, renders: list[str] +) -> None: + config = make_config(tmp_path, snapshot_interval_seconds=3600) + adapter = DGDRRunOutputAdapter(config, workload=None) + adapter.start() + for index in range(100): + adapter.on_candidate(record(f"p{index:03d}", score=float(index))) + + started = time.monotonic() + adapter.close(phase=RunPhase.FAILED, message="boom", error="OOM") + assert time.monotonic() - started < 5 + + snapshot = load(tmp_path) + assert snapshot["run"] == { + "phase": "Failed", + "terminal": True, + "message": "boom", + "error": "OOM", + } + assert tags(snapshot) == ["p099", "p098", "p097"] + assert snapshot["progress"]["evaluated"] == 100 + assert adapter.snapshot_writes == 2 # initial + terminal, nothing in between + + +def test_the_terminal_snapshot_contains_every_update_accepted_before_close( + tmp_path: Path, renders: list[str] +) -> None: + config = make_config(tmp_path, snapshot_interval_seconds=3600) + with DGDRRunOutputAdapter(config, workload=None) as adapter: + for index in range(50): + adapter.on_candidate(record(f"w{index}", score=1.0 + index)) + adapter.on_candidate(record("winner", score=1000.0)) + + snapshot = load(tmp_path) + assert snapshot["run"]["terminal"] is True + assert tags(snapshot)[0] == "winner" + + +def test_callbacks_never_wait_for_a_slow_writer( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + release = threading.Event() + + def slow_render(candidate: Any, *args: Any, **kwargs: Any) -> str: + release.wait(5) + return "kind: DynamoGraphDeployment\n" + + monkeypatch.setattr(adapter_module, "render_dgd", slow_render) + adapter = DGDRRunOutputAdapter(make_config(tmp_path), workload=None) + adapter.start() + adapter.on_candidate(record("a", 1.0)) + time.sleep(0.1) # the writer is now stuck rendering + + started = time.monotonic() + for index in range(1000): + adapter.on_candidate(record(f"q{index}", score=float(index))) + assert time.monotonic() - started < 2 + release.set() + adapter.close() + # Nothing was dropped while the writer was stuck. + assert load(tmp_path)["progress"]["evaluated"] == 1001 + + +def test_updates_after_close_are_ignored_and_close_is_idempotent( + tmp_path: Path, renders: list[str] +) -> None: + adapter = DGDRRunOutputAdapter(make_config(tmp_path), workload=None) + adapter.on_candidate(record("a", 1.0)) + adapter.close(phase=RunPhase.SUCCEEDED) + written = (tmp_path / SNAPSHOT_FILE_NAME).read_text() + writes = adapter.snapshot_writes + + adapter.on_candidate(record("late", 99.0)) + adapter.on_round(9, []) + adapter.close(phase=RunPhase.FAILED, error="ignored") + + assert (tmp_path / SNAPSHOT_FILE_NAME).read_text() == written + assert adapter.snapshot_writes == writes + + +def test_close_rejects_a_non_terminal_phase(tmp_path: Path) -> None: + adapter = DGDRRunOutputAdapter(make_config(tmp_path), workload=None) + with pytest.raises(ValueError, match="terminal"): + adapter.close(phase=RunPhase.RUNNING) + + +# -- failure handling --------------------------------------------------------- + + +def test_context_manager_records_success_and_failure( + tmp_path: Path, renders: list[str] +) -> None: + ok = tmp_path / "ok" + with DGDRRunOutputAdapter(make_config(ok), workload=None): + pass + assert load(ok)["run"]["phase"] == "Succeeded" + + failed = tmp_path / "failed" + with pytest.raises(RuntimeError, match="search crashed"): + with DGDRRunOutputAdapter(make_config(failed), workload=None): + raise RuntimeError("search crashed") + snapshot = load(failed) + assert snapshot["run"]["phase"] == "Failed" + assert snapshot["run"]["error"] == "RuntimeError" + + +def test_a_terminal_write_failure_is_raised_not_swallowed( + tmp_path: Path, renders: list[str], monkeypatch: pytest.MonkeyPatch +) -> None: + adapter = DGDRRunOutputAdapter(make_config(tmp_path), workload=None) + adapter.start() + + def broken(*_: Any, **__: Any) -> None: + raise OSError("disk full") + + monkeypatch.setattr(adapter_module, "write_snapshot", broken) + with pytest.raises(OSError, match="disk full"): + adapter.close() + + +def test_a_failed_periodic_write_is_retried_with_the_latest_state( + tmp_path: Path, renders: list[str], monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(adapter_module, "_WRITE_RETRY_BACKOFF", 0.02) + adapter = DGDRRunOutputAdapter(make_config(tmp_path), workload=None) + adapter.start() + real = adapter_module.write_snapshot + attempts = {"count": 0} + + def flaky(directory: Path, snapshot: Any) -> Path: + attempts["count"] += 1 + if attempts["count"] == 1: + raise OSError("transient") + return real(directory, snapshot) + + monkeypatch.setattr(adapter_module, "write_snapshot", flaky) + adapter.on_candidate(record("a", 1.0)) + wait_for(lambda: tags(load(tmp_path)) == ["a"]) + adapter.close() + assert attempts["count"] >= 2 + + +def test_a_candidate_that_cannot_be_read_does_not_break_the_search( + tmp_path: Path, renders: list[str] +) -> None: + adapter = DGDRRunOutputAdapter(make_config(tmp_path), workload=None) + adapter.on_candidate(SimpleNamespace(status="feasible")) # type: ignore[arg-type] + adapter.on_candidate(record("ok", 1.0)) + adapter.close() + assert tags(load(tmp_path)) == ["ok"] + + +# -- configuration and plugin ------------------------------------------------- + + +def test_pareto_objectives_are_rejected_explicitly(tmp_path: Path) -> None: + with pytest.raises(NotImplementedError, match="scalar"): + DGDRRunOutputAdapter(make_config(tmp_path), workload=None, is_pareto=True) + + +@pytest.mark.parametrize( + "overrides", + [ + {"max_candidates": 0}, + {"snapshot_interval_seconds": -1}, + {"name": "Not_DNS"}, + {"num_gpus_per_node": 0}, + {"unknown_field": 1}, + ], +) +def test_invalid_configuration_is_rejected( + tmp_path: Path, overrides: dict[str, Any] +) -> None: + with pytest.raises(ValueError): + make_config(tmp_path, **overrides) + + +def test_there_is_no_hardcoded_candidate_ceiling( + tmp_path: Path, renders: list[str] +) -> None: + adapter = DGDRRunOutputAdapter(make_config(tmp_path, max_candidates=200), None) + for index in range(150): + adapter.on_candidate(record(f"p{index:03d}", score=float(index))) + adapter.close() + assert len(load(tmp_path)["candidates"]) == 150 + + +def test_a_snapshot_directory_is_required() -> None: + config = DGDRRunOutputConfig( + name="sweep", + runtime_image="nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.2.3", + num_gpus_per_node=8, + ) + with pytest.raises(ValueError, match="snapshot directory"): + DGDRRunOutputAdapter(config, workload=None) + + +def test_the_plugin_is_discoverable_and_builds_a_live_adapter(tmp_path: Path) -> None: + plugin = create_adapter() + assert plugin.name == "dgdr_run" + live = plugin.live( + { + "name": "sweep", + "snapshot_dir": str(tmp_path), + "runtime_image": "nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.2.3", + "num_gpus_per_node": 8, + }, + workload=None, + ) + assert isinstance(live, DGDRRunOutputAdapter) + + +def test_plugin_subscribe_validates_config_before_search() -> None: + plugin = adapter_module.DGDRRunOutputPlugin() + good = { + "name": "run", + "runtime_image": "nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.2.3", + "num_gpus_per_node": 8, + } + + assert plugin.subscribe(good) is None + with pytest.raises(ValueError, match="num_gpus_per_node"): + plugin.subscribe({**good, "num_gpus_per_node": 0}) + with pytest.raises(ValueError, match="canonical"): + plugin.subscribe( + {**good, "runtime_image": "nvcr.io/nvidia/ai-dynamo/vllm-runtime:latest"} + ) + + +def test_plugin_write_publishes_the_final_selection_as_a_terminal_snapshot( + tmp_path: Path, renders: list[str], monkeypatch: pytest.MonkeyPatch +) -> None: + class FakeSearchConfig: + @staticmethod + def model_validate(_: Any) -> Any: + return SimpleNamespace(workload="workload") + + monkeypatch.setattr(adapter_module, "SmartSearchConfig", FakeSearchConfig) + result = SimpleNamespace( + views=SimpleNamespace(pareto_front=[]), + provenance=SimpleNamespace(config={}), + selected_candidates=[ + record("a", 9.0), + record("b", 5.0), + ], + ) + written = create_adapter().write( + { + "name": "sweep", + "runtime_image": "nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.2.3", + "num_gpus_per_node": 8, + }, + result=result, # type: ignore[arg-type] + output_dir=tmp_path, + ) + + assert written == [Path(SNAPSHOT_FILE_NAME)] + snapshot = load(tmp_path) + assert snapshot["run"]["terminal"] is True + assert tags(snapshot) == ["a", "b"] + + +def test_plugin_write_keeps_sweepers_selection_order( + tmp_path: Path, renders: list[str], fake_search_config: None +) -> None: + # Sweeper's selection can be goal-dependent (e.g. fewest GPUs first), so the + # adapter must not re-rank it by score. + result = SimpleNamespace( + views=SimpleNamespace(pareto_front=[]), + provenance=SimpleNamespace(config={}), + selected_candidates=[record("low", 1.0), record("high", 9.0)], + counts=SimpleNamespace(evaluated=2), + ) + create_adapter().write(PLUGIN_CONFIG, result=result, output_dir=tmp_path) # type: ignore[arg-type] + assert tags(load(tmp_path)) == ["low", "high"] + + +def test_plugin_write_refuses_pareto_and_empty_results(tmp_path: Path) -> None: + config = { + "name": "sweep", + "runtime_image": "nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.2.3", + "num_gpus_per_node": 8, + } + pareto = SimpleNamespace(views=SimpleNamespace(pareto_front=[object()])) + with pytest.raises(NotImplementedError): + create_adapter().write(config, result=pareto, output_dir=tmp_path) # type: ignore[arg-type] + empty = SimpleNamespace( + views=SimpleNamespace(pareto_front=[]), selected_candidates=[] + ) + with pytest.raises(CandidateMaterializationError): + create_adapter().write(config, result=empty, output_dir=tmp_path) # type: ignore[arg-type] + + +PLUGIN_CONFIG = { + "name": "sweep", + "runtime_image": "nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.2.3", + "num_gpus_per_node": 8, + "snapshot_interval_seconds": 0, +} + + +def test_plugin_subscribe_with_a_context_publishes_live( + tmp_path: Path, renders: list[str] +) -> None: + config = {**PLUGIN_CONFIG, "snapshot_dir": str(tmp_path)} + callbacks = create_adapter().subscribe( + config, context=SimpleNamespace(workload="workload") + ) + + assert callbacks is not None + assert callbacks.on_candidate is not None + assert callbacks.on_round is not None + assert load(tmp_path)["run"] == {"phase": "Running", "terminal": False} + + callbacks.on_candidate(record("a", 9.0)) + wait_for(lambda: tags(load(tmp_path)) == ["a"]) + assert callbacks.on_complete is not None + callbacks.on_complete() + + +def test_plugin_subscribe_without_context_or_snapshot_dir_only_validates( + tmp_path: Path, +) -> None: + plugin = create_adapter() + with_dir = {**PLUGIN_CONFIG, "snapshot_dir": str(tmp_path)} + assert plugin.subscribe(with_dir) is None + assert plugin.subscribe(with_dir, context=SimpleNamespace(workload=None)) is None + assert ( + plugin.subscribe(PLUGIN_CONFIG, context=SimpleNamespace(workload="workload")) + is None + ) + assert not (tmp_path / SNAPSHOT_FILE_NAME).exists() + + +def _final_result(evaluated: int = 42) -> Any: + return SimpleNamespace( + views=SimpleNamespace(pareto_front=[]), + provenance=SimpleNamespace(config={}), + selected_candidates=[record("a", 9.0)], + counts=SimpleNamespace(evaluated=evaluated), + ) + + +@pytest.fixture +def fake_search_config(monkeypatch: pytest.MonkeyPatch) -> None: + class FakeSearchConfig: + @staticmethod + def model_validate(_: Any) -> Any: + return SimpleNamespace(workload="workload") + + monkeypatch.setattr(adapter_module, "SmartSearchConfig", FakeSearchConfig) + + +def test_plugin_write_reports_no_artifact_for_a_snapshot_outside_output_dir( + tmp_path: Path, renders: list[str], fake_search_config: None +) -> None: + shared = tmp_path / "shared" + shared.mkdir() + out = tmp_path / "out" + out.mkdir() + + written = create_adapter().write( + {**PLUGIN_CONFIG, "snapshot_dir": str(shared)}, + result=_final_result(), + output_dir=out, + ) + + assert written == [] + assert load(shared)["run"]["terminal"] is True + assert not (out / SNAPSHOT_FILE_NAME).exists() + + +def test_a_transient_materialization_failure_is_retried_and_recovers( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + attempts: list[int] = [] + + def flaky_render(candidate: Any, workload: Any, options: Any, **kwargs: Any) -> str: + attempts.append(1) + if len(attempts) == 1: + raise CandidateMaterializationError("cold start") + return "apiVersion: nvidia.com/v1beta1\nkind: DynamoGraphDeployment\n" + + monkeypatch.setattr(adapter_module, "render_dgd", flaky_render) + adapter = DGDRRunOutputAdapter(make_config(tmp_path), workload=None) + adapter.start() + adapter.on_candidate(record("a", 5.0)) + wait_for(lambda: len(load(tmp_path)["candidates"]) == 1) + assert load(tmp_path)["candidates"][0]["outcome"] == "materialization_failed" + + adapter.on_round(1, []) # any later write retries the failed render + wait_for(lambda: load(tmp_path)["candidates"][0]["outcome"] == "materialized") + adapter.close() + + (candidate,) = load(tmp_path)["candidates"] + assert candidate["outcome"] == "materialized" + assert "error" not in candidate + assert len(attempts) == 2 + + +def test_plugin_write_without_a_live_snapshot_starts_at_round_zero( + tmp_path: Path, renders: list[str], fake_search_config: None +) -> None: + create_adapter().write(PLUGIN_CONFIG, result=_final_result(2), output_dir=tmp_path) + assert load(tmp_path)["progress"] == {"round": 0, "evaluated": 2} + + +# -- lifecycle: how the search ended -------------------------------------------- + + +def _subscribed(tmp_path: Path) -> Any: + callbacks = create_adapter().subscribe( + {**PLUGIN_CONFIG, "snapshot_dir": str(tmp_path)}, + context=SimpleNamespace(workload="workload"), + ) + assert callbacks is not None + return callbacks + + +def test_a_failed_search_leaves_a_terminal_failed_snapshot( + tmp_path: Path, renders: list[str] +) -> None: + callbacks = _subscribed(tmp_path) + callbacks.on_candidate(record("a", 9.0)) + callbacks.on_round(2, []) + + callbacks.on_failure(RuntimeError("replay crashed")) + + snapshot = load(tmp_path) + assert snapshot["run"] == { + "phase": "Failed", + "terminal": True, + "message": "replay crashed", + "error": "RuntimeError", + } + assert tags(snapshot) == ["a"] + assert snapshot["progress"]["round"] == 2 + + +def test_a_long_failure_message_is_truncated( + tmp_path: Path, renders: list[str] +) -> None: + callbacks = _subscribed(tmp_path) + callbacks.on_failure(RuntimeError("x" * 5000)) + assert len(load(tmp_path)["run"]["message"]) == adapter_module._MAX_ERROR_MESSAGE + + +def test_a_search_that_finds_nothing_is_recorded_as_failed( + tmp_path: Path, renders: list[str] +) -> None: + callbacks = _subscribed(tmp_path) + callbacks.on_candidate(record("infeasible", 1.0, status="infeasible")) + + callbacks.on_complete() + + run = load(tmp_path)["run"] + assert run["phase"] == "Failed" + assert run["terminal"] is True + assert run["message"] == "no feasible candidate found" + + +def test_a_completed_search_flushes_stops_the_writer_and_stays_non_terminal( + tmp_path: Path, renders: list[str] +) -> None: + config = make_config(tmp_path, snapshot_interval_seconds=60) + adapter = DGDRRunOutputAdapter(config, workload=None) + adapter.start() + adapter.on_candidate(record("a", 9.0)) + adapter.on_round(3, []) + + adapter.finish_search() + + snapshot = load(tmp_path) + assert snapshot["run"] == {"phase": "Running", "terminal": False} + assert tags(snapshot) == ["a"] + assert snapshot["progress"] == {"round": 3, "evaluated": 1} + # The writer is gone: nothing can overwrite the terminal snapshot written next. + assert adapter._thread is not None and not adapter._thread.is_alive() + writes = adapter.snapshot_writes + adapter.on_round(4, []) + time.sleep(0.1) + assert adapter.snapshot_writes == writes + + +def test_the_terminal_snapshot_after_a_completed_search_keeps_the_live_progress( + tmp_path: Path, renders: list[str], fake_search_config: None +) -> None: + callbacks = _subscribed(tmp_path) + callbacks.on_candidate(record("a", 9.0)) + callbacks.on_round(3, []) + callbacks.on_complete() + + create_adapter().write( + {**PLUGIN_CONFIG, "snapshot_dir": str(tmp_path)}, + result=_final_result(1), + output_dir=tmp_path, + ) + + snapshot = load(tmp_path) + assert snapshot["run"]["phase"] == "Succeeded" + assert snapshot["progress"] == {"round": 3, "evaluated": 1} + + +def test_failure_after_completion_is_a_noop_and_completion_after_failure_too( + tmp_path: Path, renders: list[str] +) -> None: + callbacks = _subscribed(tmp_path) + callbacks.on_candidate(record("a", 9.0)) + callbacks.on_failure(RuntimeError("first")) + callbacks.on_complete() + callbacks.on_failure(RuntimeError("second")) + assert load(tmp_path)["run"]["message"] == "first" + + +# -- robustness ----------------------------------------------------------------- + + +def test_a_broken_renderer_leaves_a_terminal_failed_snapshot( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + def broken(*_: Any, **__: Any) -> str: + raise RuntimeError("renderer not installed") + + monkeypatch.setattr(adapter_module, "render_dgd", broken) + adapter = DGDRRunOutputAdapter(make_config(tmp_path), workload=None) + adapter.on_candidate(record("a", 9.0)) + + with pytest.raises(RuntimeError, match="renderer not installed"): + adapter.close() + + snapshot = load(tmp_path) + assert snapshot["run"]["phase"] == "Failed" + assert snapshot["run"]["terminal"] is True + assert snapshot["run"]["error"] == "RuntimeError" + assert all(c["outcome"] != "materialization_failed" for c in snapshot["candidates"]) + + +def test_non_finite_scores_and_objectives_stay_valid_for_the_publisher( + tmp_path: Path, renders: list[str] +) -> None: + adapter = DGDRRunOutputAdapter(make_config(tmp_path), workload=None) + adapter.on_candidate(record("a", float("-inf"), objectives={"e2e": float("nan")})) + adapter.on_candidate(record("b", float("nan"))) + adapter.close() + + text = (tmp_path / SNAPSHOT_FILE_NAME).read_text() + assert ( + "inf" not in text.lower().replace("information", "") + and "nan" not in text.lower() + ) + metrics = { + c["parameters"]["tag"]: c["metrics"] for c in load(tmp_path)["candidates"] + } + assert metrics["a"]["score"] is None and metrics["a"]["objectives"] == {"e2e": None} + assert metrics["b"]["score"] is None + + +def test_a_candidate_error_message_is_bounded( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + def failing(*_: Any, **__: Any) -> str: + raise CandidateMaterializationError("x" * 5000) + + monkeypatch.setattr(adapter_module, "render_dgd", failing) + adapter = DGDRRunOutputAdapter(make_config(tmp_path), workload=None) + adapter.on_candidate(record("a", 9.0)) + adapter.close() + + (candidate,) = load(tmp_path)["candidates"] + assert 0 < len(candidate["error"]) <= 500 + + +def test_subscribe_does_not_publish_live_without_lifecycle_callbacks( + tmp_path: Path, renders: list[str], monkeypatch: pytest.MonkeyPatch +) -> None: + @dataclass + class LegacyCallbacks: + on_candidate: Any = None + on_round: Any = None + + monkeypatch.setattr( + adapter_module, "RecommendationOutputCallbacks", LegacyCallbacks + ) + callbacks = create_adapter().subscribe( + {**PLUGIN_CONFIG, "snapshot_dir": str(tmp_path)}, + context=SimpleNamespace(workload="workload"), + ) + assert callbacks is None + assert not (tmp_path / SNAPSHOT_FILE_NAME).exists() + + +def test_a_persistent_write_failure_does_not_spin_the_writer( + tmp_path: Path, renders: list[str], monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(adapter_module, "_WRITE_RETRY_BACKOFF", 0.2) + adapter = DGDRRunOutputAdapter(make_config(tmp_path), workload=None) + adapter.start() + attempts = {"count": 0} + + def failing(directory: Path, snapshot: Any) -> Path: + attempts["count"] += 1 + raise OSError("disk full") + + monkeypatch.setattr(adapter_module, "write_snapshot", failing) + adapter.on_candidate(record("a", 1.0)) + time.sleep(0.6) + seen = attempts["count"] + monkeypatch.undo() + adapter.close() + assert 1 <= seen <= 5 + + +def test_plugin_write_keeps_the_objectives_of_the_selected_candidates( + tmp_path: Path, renders: list[str], fake_search_config: None +) -> None: + result = _final_result(1) + result.selected_candidates = [record("a", 9.0, objectives={"throughput": 12.5})] + create_adapter().write(PLUGIN_CONFIG, result=result, output_dir=tmp_path) + (candidate,) = load(tmp_path)["candidates"] + assert candidate["metrics"]["objectives"] == {"throughput": 12.5} + + +def test_the_renderer_receives_the_candidates_gpu_count( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + seen: list[Any] = [] + + def capture(candidate: Any, *_: Any, **kwargs: Any) -> str: + seen.append(candidate) + return "kind: DynamoGraphDeployment\n" + + monkeypatch.setattr(adapter_module, "render_dgd", capture) + adapter = DGDRRunOutputAdapter(make_config(tmp_path), workload=None) + adapter.on_candidate(record("a", 9.0, gpus=4)) + adapter.close() + + (candidate,) = seen + assert candidate.used_gpus == 4 + assert candidate.config["tag"] == "a" diff --git a/components/src/dynamo/aisimulate/tests/dgdr_run/test_snapshot.py b/components/src/dynamo/aisimulate/tests/dgdr_run/test_snapshot.py new file mode 100644 index 000000000000..6ea05b6536e2 --- /dev/null +++ b/components/src/dynamo/aisimulate/tests/dgdr_run/test_snapshot.py @@ -0,0 +1,148 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from pathlib import Path + +import pytest +import yaml + +from dynamo.aisimulate.output.dgdr_run.snapshot import ( + SCHEMA_VERSION, + SNAPSHOT_FILE_NAME, + CandidateOutcome, + DGDRRunSnapshot, + RunPhase, + SnapshotCandidate, + write_snapshot, +) + +pytestmark = [ + pytest.mark.unit, + pytest.mark.gpu_0, + pytest.mark.pre_merge, + pytest.mark.planner, + pytest.mark.parallel, +] + + +def materialized(candidate_id: str = "evaluated-point-a") -> SnapshotCandidate: + return SnapshotCandidate( + id=candidate_id, + outcome=CandidateOutcome.MATERIALIZED, + parameters={"tp": 4}, + metrics={"score": 1.5, "usedGpus": 4}, + manifest="kind: DynamoGraphDeployment\n", + ) + + +def test_wire_format_of_a_running_snapshot() -> None: + payload = DGDRRunSnapshot( + phase=RunPhase.RUNNING, + round_no=7, + evaluated=48, + candidates=(materialized(),), + ).to_dict() + + assert payload["schemaVersion"] == SCHEMA_VERSION + assert payload["run"] == {"phase": "Running", "terminal": False} + assert payload["progress"] == {"round": 7, "evaluated": 48} + assert payload["candidates"] == [ + { + "id": "evaluated-point-a", + "outcome": "materialized", + "parameters": {"tp": 4}, + "metrics": {"score": 1.5, "usedGpus": 4}, + "manifest": "kind: DynamoGraphDeployment\n", + } + ] + assert payload["timestamp"].endswith("Z") + + +def test_terminal_snapshot_carries_the_outcome() -> None: + payload = DGDRRunSnapshot( + phase=RunPhase.FAILED, + round_no=2, + evaluated=9, + message="stopped", + error="OOM", + ).to_dict() + assert payload["run"] == { + "phase": "Failed", + "terminal": True, + "message": "stopped", + "error": "OOM", + } + assert payload["candidates"] == [] + + +def test_a_failed_candidate_carries_an_error_and_no_manifest() -> None: + failed = SnapshotCandidate( + id="evaluated-point-b", + outcome=CandidateOutcome.MATERIALIZATION_FAILED, + error="cannot render", + ) + assert failed.to_dict() == { + "id": "evaluated-point-b", + "outcome": "materialization_failed", + "parameters": {}, + "metrics": {}, + "error": "cannot render", + } + + +@pytest.mark.parametrize( + "kwargs", + [ + {"id": "", "outcome": CandidateOutcome.MATERIALIZED, "manifest": "x"}, + {"id": "a", "outcome": CandidateOutcome.MATERIALIZED}, + { + "id": "a", + "outcome": CandidateOutcome.MATERIALIZED, + "manifest": "x", + "error": "y", + }, + {"id": "a", "outcome": CandidateOutcome.MATERIALIZATION_FAILED}, + { + "id": "a", + "outcome": CandidateOutcome.MATERIALIZATION_FAILED, + "error": "y", + "manifest": "x", + }, + ], +) +def test_inconsistent_candidates_are_rejected(kwargs: dict) -> None: + with pytest.raises(ValueError): + SnapshotCandidate(**kwargs) + + +def test_write_replaces_the_file_atomically_and_leaves_no_temporary_files( + tmp_path: Path, +) -> None: + for round_no in range(2): + path = write_snapshot( + tmp_path, + DGDRRunSnapshot(phase=RunPhase.RUNNING, round_no=round_no, evaluated=0), + ) + assert path == tmp_path / SNAPSHOT_FILE_NAME + assert yaml.safe_load(path.read_text())["progress"]["round"] == round_no + + assert [p.name for p in tmp_path.iterdir()] == [SNAPSHOT_FILE_NAME] + + +def test_the_manifest_survives_a_yaml_round_trip_byte_for_byte( + tmp_path: Path, +) -> None: + manifest = "apiVersion: nvidia.com/v1beta1\nkind: DynamoGraphDeployment\nspec:\n a: 'x: y'\n" + candidate = SnapshotCandidate( + id="evaluated-point-c", + outcome=CandidateOutcome.MATERIALIZED, + manifest=manifest, + ) + path = write_snapshot( + tmp_path, + DGDRRunSnapshot( + phase=RunPhase.RUNNING, round_no=0, evaluated=1, candidates=(candidate,) + ), + ) + (loaded,) = yaml.safe_load(path.read_text())["candidates"] + assert loaded["manifest"] == manifest diff --git a/components/src/dynamo/common/container_images.py b/components/src/dynamo/common/container_images.py new file mode 100644 index 000000000000..4e00b491a297 --- /dev/null +++ b/components/src/dynamo/common/container_images.py @@ -0,0 +1,40 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Published Dynamo container-image naming conventions.""" + +BACKEND_IMAGE_NAMES: dict[str, str] = { + "vllm": "vllm-runtime", + "sglang": "sglang-runtime", + "trtllm": "tensorrtllm-runtime", +} + +PLANNER_IMAGE_NAME = "dynamo-planner" + + +def _replace_image_name(image_ref: str, new_name: str) -> str: + """Replace the image name while preserving its registry, tag and digest.""" + slash_idx = image_ref.rfind("/") + prefix = image_ref[: slash_idx + 1] if slash_idx >= 0 else "" + suffix = image_ref[slash_idx + 1 :] + name_and_tag, has_digest, digest = suffix.partition("@") + colon_idx = name_and_tag.rfind(":") + tag = name_and_tag[colon_idx:] if colon_idx >= 0 else "" + digest_suffix = f"@{digest}" if has_digest else "" + return f"{prefix}{new_name}{tag}{digest_suffix}" + + +def derive_planner_image(source_image: str) -> str: + """Derive the planner image that belongs to a published Dynamo image.""" + return _replace_image_name(source_image, PLANNER_IMAGE_NAME) + + +def derive_backend_image(source_image: str, backend: str) -> str: + """Derive one backend runtime image from a published Dynamo image.""" + backend_image_name = BACKEND_IMAGE_NAMES.get(backend) + if backend_image_name is None: + raise ValueError( + f"Cannot derive backend image for unknown backend '{backend}'. " + f"Supported backends: {list(BACKEND_IMAGE_NAMES.keys())}" + ) + return _replace_image_name(source_image, backend_image_name) diff --git a/components/src/dynamo/profiler/utils/profile_common.py b/components/src/dynamo/profiler/utils/profile_common.py index 4fc42e335c8d..c01b06c312fc 100644 --- a/components/src/dynamo/profiler/utils/profile_common.py +++ b/components/src/dynamo/profiler/utils/profile_common.py @@ -23,6 +23,8 @@ import pandas as pd +from dynamo.common.container_images import derive_backend_image as derive_backend_image +from dynamo.common.container_images import derive_planner_image as derive_planner_image from dynamo.planner.config.planner_config import PlannerPreDeploymentSweepMode from dynamo.profiler.utils.config_modifiers.parallelization_mapping import ( PickedParallelConfig, @@ -36,82 +38,6 @@ logger = logging.getLogger(__name__) -# --------------------------------------------------------------------------- -# Published container image naming conventions -# --------------------------------------------------------------------------- - -# Mapping from backend name to the image-name component of the published -# backend runtime image. -# e.g. vllm → nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.1.1 -BACKEND_IMAGE_NAMES: dict[str, str] = { - "vllm": "vllm-runtime", - "sglang": "sglang-runtime", - "trtllm": "tensorrtllm-runtime", -} - -PLANNER_IMAGE_NAME = "dynamo-planner" - - -def _replace_image_name(image_ref: str, new_name: str) -> str: - """Replace the image name component in a Docker image reference. - - Preserves the registry path prefix and tag suffix, only replacing the - last ``/``-delimited component (before any ``:tag``). - """ - slash_idx = image_ref.rfind("/") - prefix = image_ref[: slash_idx + 1] if slash_idx >= 0 else "" - suffix = image_ref[slash_idx + 1 :] - name_and_tag, has_digest, digest = suffix.partition("@") - colon_idx = name_and_tag.rfind(":") - tag = name_and_tag[colon_idx:] if colon_idx >= 0 else "" - digest_suffix = f"@{digest}" if has_digest else "" - return f"{prefix}{new_name}{tag}{digest_suffix}" - - -def derive_planner_image(profiler_image: str) -> str: - """Derive the planner service image from the profiler image reference.""" - return _replace_image_name(profiler_image, PLANNER_IMAGE_NAME) - - -def derive_backend_image(profiler_image: str, backend: str) -> str: - """Derive the backend worker image from the profiler image. - - Replaces the image name (the last ``/``-delimited component, before any - ``:tag``) with the backend-specific runtime image name, preserving the - registry path and tag unchanged. - - Examples:: - - derive_backend_image( - "nvcr.io/nvidia/ai-dynamo/dynamo-planner:1.1.1", "vllm" - ) - # → "nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.1.1" - - derive_backend_image("myregistry.io/sglang-runtime:1.1.1", "sglang") - # → "myregistry.io/sglang-runtime:1.1.1" - - Args: - profiler_image: Any Docker image reference of the form - ``[REGISTRY/]NAME[:TAG]``. - backend: The resolved backend type (``'vllm'``, ``'sglang'``, or - ``'trtllm'``). - - Returns: - The backend container image string. - - Raises: - ValueError: If *backend* is not a recognised backend. - """ - backend_image_name = BACKEND_IMAGE_NAMES.get(backend) - if backend_image_name is None: - raise ValueError( - f"Cannot derive backend image for unknown backend '{backend}'. " - f"Supported backends: {list(BACKEND_IMAGE_NAMES.keys())}" - ) - - return _replace_image_name(profiler_image, backend_image_name) - - # --------------------------------------------------------------------------- # Operational defaults not part of DynamoGraphDeploymentRequestSpec # --------------------------------------------------------------------------- diff --git a/deploy/helm/charts/platform/README.md b/deploy/helm/charts/platform/README.md index 28084f1829be..eb9641c2bd20 100644 --- a/deploy/helm/charts/platform/README.md +++ b/deploy/helm/charts/platform/README.md @@ -454,7 +454,7 @@ Kubernetes: `>=1.30.0-0` | dynamo-operator.grovePreflight.deploymentName | string | `""` | External Grove Deployment name. Requires namespace; empty discovers by container name. | | dynamo-operator.enabled | bool | `true` | Whether to enable the Dynamo Kubernetes operator deployment | | dynamo-operator.upgradeCRD | bool | `true` | Whether the cluster-wide operator applies CRDs from its image through the crd-apply init container. When false, the chart installs no CRDs; apply them separately before starting a cluster-wide operator. Namespace-restricted installations must set this to false and use the CRDs managed by the cluster-wide operator. | -| dynamo-operator.dgdrV1Beta2.enabled | bool | `false` | EXPERIMENTAL, DEVELOPMENT CLUSTERS ONLY: Serve the unfinished DGDR v1beta2 Request API with conversion and install the Run and Candidate APIs. Run and Candidate objects are persisted in etcd; incompatible schema changes may make them unreadable or block future CRD upgrades. The operator does not reconcile v1beta2 requests. | +| dynamo-operator.dgdrV1Beta2.enabled | bool | `false` | EXPERIMENTAL, DEVELOPMENT CLUSTERS ONLY: Serve the unfinished DGDR v1beta2 Request API with conversion and install the Run and Candidate APIs. Run and Candidate objects are persisted in etcd; incompatible schema changes may make them unreadable or block future CRD upgrades. When true, the operator also reconciles v1beta2 Requests into Runs; the controller that executes a Run is not part of the operator yet. | | dynamo-operator.natsAddr | string | `""` | NATS server address for operator communication. When empty, the operator uses bundled NATS only if global.nats.install=true; otherwise NATS is not configured. Format: `nats://hostname:4222` | | dynamo-operator.etcdAddr | string | `""` | etcd server address for an external etcd instance. Only needed when using external etcd without the bundled subchart. Format: `http://hostname:2379` or `https://hostname:2379` | | dynamo-operator.modelExpressURL | string | `""` | URL for the Model Express server if not deployed by this helm chart. This is ignored if Model Express server is installed by this helm chart (global.model-express.enabled is true). | diff --git a/deploy/helm/charts/platform/components/operator/files/role.yaml b/deploy/helm/charts/platform/components/operator/files/role.yaml index 1ec01f3f9264..c33fb4afee55 100644 --- a/deploy/helm/charts/platform/components/operator/files/role.yaml +++ b/deploy/helm/charts/platform/components/operator/files/role.yaml @@ -24,9 +24,10 @@ rules: - "" resources: - nodes - - pods/log verbs: - get + - list + - watch - apiGroups: - "" resources: @@ -48,15 +49,31 @@ rules: - list - patch - watch +- apiGroups: + - "" + resources: + - pods/log + verbs: + - get - apiGroups: - "" resources: - secrets + verbs: + - create + - get + - list + - update + - watch +- apiGroups: + - "" + resources: - serviceaccounts verbs: - create - get - list + - patch - update - watch - apiGroups: @@ -296,6 +313,7 @@ rules: resources: - dynamocomponentdeployments/status - dynamographdeploymentrequests/status + - dynamographdeploymentruns/status - dynamographdeployments/status - dynamographdeploymentscalingadapters/status - dynamomodels/status @@ -304,6 +322,16 @@ rules: - get - patch - update +- apiGroups: + - nvidia.com + resources: + - dynamographdeploymentruns + verbs: + - create + - delete + - get + - list + - watch - apiGroups: - nvidia.com resources: @@ -342,6 +370,7 @@ rules: - delete - get - list + - patch - update - watch - apiGroups: @@ -352,6 +381,7 @@ rules: - create - get - list + - patch - update - watch - apiGroups: diff --git a/deploy/helm/charts/platform/components/operator/templates/deployment.yaml b/deploy/helm/charts/platform/components/operator/templates/deployment.yaml index 8a4ea7dd9080..241a98346a27 100644 --- a/deploy/helm/charts/platform/components/operator/templates/deployment.yaml +++ b/deploy/helm/charts/platform/components/operator/templates/deployment.yaml @@ -92,6 +92,10 @@ spec: {{- with .Values.controllerManager.manager.dgdrDefaultImage }} - {{ printf "--dgdr-default-image=%s" . | quote }} {{- end }} + {{- with .Values.controllerManager.manager.dgdrAISimulateDefaultImage }} + - {{ printf "--dgdr-aisimulate-default-image=%s" . | quote }} + {{- end }} + - --dgdr-v1beta2-enabled={{ .Values.dgdrV1Beta2.enabled }} command: - /manager env: diff --git a/deploy/helm/charts/platform/components/operator/templates/webhook-configuration.yaml b/deploy/helm/charts/platform/components/operator/templates/webhook-configuration.yaml index 5f6880fe46e6..770627f942b3 100644 --- a/deploy/helm/charts/platform/components/operator/templates/webhook-configuration.yaml +++ b/deploy/helm/charts/platform/components/operator/templates/webhook-configuration.yaml @@ -150,6 +150,7 @@ webhooks: namespace: {{ .Release.Namespace }} path: /validate-nvidia-com-v1beta1-dynamographdeploymentrequest failurePolicy: {{ .Values.webhook.failurePolicy }} + matchPolicy: Equivalent name: vdynamographdeploymentrequest.kb.io {{- if .Values.webhook.namespaceSelector }} namespaceSelector: @@ -274,6 +275,7 @@ webhooks: namespace: {{ .Release.Namespace }} path: /mutate-nvidia-com-v1beta1-dynamographdeploymentrequest failurePolicy: {{ .Values.webhook.failurePolicy }} + matchPolicy: Equivalent name: mdynamographdeploymentrequestv1beta1.kb.io {{- if .Values.webhook.namespaceSelector }} namespaceSelector: diff --git a/deploy/helm/charts/platform/components/operator/values.yaml b/deploy/helm/charts/platform/components/operator/values.yaml index c526a93bebde..a3ac9a0d7da5 100644 --- a/deploy/helm/charts/platform/components/operator/values.yaml +++ b/deploy/helm/charts/platform/components/operator/values.yaml @@ -36,7 +36,9 @@ dgdrV1Beta2: # Candidate objects are persisted in etcd; incompatible schema changes may make # them unreadable or block future CRD upgrades. When false, the CRD init container # installs the legacy Request schemas without v1beta2 conversion envelopes and - # omits Run and Candidate. The operator does not reconcile v1beta2 requests. + # omits Run and Candidate. When true, the operator also reconciles v1beta2 + # Requests into Runs; the controller that executes a Run is not part of the + # operator yet. enabled: false # Environment variables to pass to operator Deployment. @@ -115,6 +117,9 @@ controllerManager: # derives dynamo-planner:, which only exists for GA # releases. dgdrDefaultImage: "" + # AISimulate runner image put into v1beta2 DGDR spec.aisimulate.image when unset. + # Empty derives nvcr.io/nvidia/ai-dynamo/dynamo-planner:. + dgdrAISimulateDefaultImage: "" resources: limits: cpu: 1024m diff --git a/deploy/helm/charts/platform/tests/namespace_restriction_deployment_test.yaml b/deploy/helm/charts/platform/tests/namespace_restriction_deployment_test.yaml index 41789e477c6b..d9855f666593 100644 --- a/deploy/helm/charts/platform/tests/namespace_restriction_deployment_test.yaml +++ b/deploy/helm/charts/platform/tests/namespace_restriction_deployment_test.yaml @@ -93,3 +93,25 @@ tests: asserts: - failedTemplate: errorPattern: "webhook.namespaceSelector must be empty" + + - it: passes the DGDR v1beta2 switch to the manager + template: charts/dynamo-operator/templates/deployment.yaml + asserts: + - contains: + path: spec.template.spec.containers[0].args + content: --dgdr-v1beta2-enabled=false + documentSelector: + path: kind + value: Deployment + + - it: enables the DGDR v1beta2 controller in the manager + template: charts/dynamo-operator/templates/deployment.yaml + set: + dynamo-operator.dgdrV1Beta2.enabled: true + asserts: + - contains: + path: spec.template.spec.containers[0].args + content: --dgdr-v1beta2-enabled=true + documentSelector: + path: kind + value: Deployment diff --git a/deploy/helm/charts/platform/tests/namespace_restriction_webhooks_test.yaml b/deploy/helm/charts/platform/tests/namespace_restriction_webhooks_test.yaml index af335066ea7a..42edcb584537 100644 --- a/deploy/helm/charts/platform/tests/namespace_restriction_webhooks_test.yaml +++ b/deploy/helm/charts/platform/tests/namespace_restriction_webhooks_test.yaml @@ -97,3 +97,23 @@ tests: asserts: - hasDocuments: count: 4 + + - it: matches DGDR requests through every served API version + template: charts/dynamo-operator/templates/webhook-configuration.yaml + asserts: + - equal: + path: webhooks[3].name + value: vdynamographdeploymentrequest.kb.io + documentIndex: 0 + - equal: + path: webhooks[3].matchPolicy + value: Equivalent + documentIndex: 0 + - equal: + path: webhooks[2].name + value: mdynamographdeploymentrequestv1beta1.kb.io + documentIndex: 1 + - equal: + path: webhooks[2].matchPolicy + value: Equivalent + documentIndex: 1 diff --git a/deploy/helm/charts/platform/values.yaml b/deploy/helm/charts/platform/values.yaml index 13c14c9c058f..fe6daeec961b 100644 --- a/deploy/helm/charts/platform/values.yaml +++ b/deploy/helm/charts/platform/values.yaml @@ -97,8 +97,9 @@ dynamo-operator: # -- EXPERIMENTAL, DEVELOPMENT CLUSTERS ONLY: Serve the unfinished DGDR v1beta2 # Request API with conversion and install the Run and Candidate APIs. Run and # Candidate objects are persisted in etcd; incompatible schema changes may make - # them unreadable or block future CRD upgrades. The operator does not reconcile - # v1beta2 requests. + # them unreadable or block future CRD upgrades. When true, the operator also + # reconciles v1beta2 Requests into Runs; the controller that executes a Run is + # not part of the operator yet. enabled: false # Environment variables to pass to operator Deployment. diff --git a/deploy/operator/Dockerfile b/deploy/operator/Dockerfile index 35f9835e5aa6..09886f121aea 100644 --- a/deploy/operator/Dockerfile +++ b/deploy/operator/Dockerfile @@ -53,6 +53,7 @@ FROM base AS builder RUN CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} go build -o manager ./cmd/main.go && \ CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} go build -o crd-apply ./cmd/crd-apply/main.go && \ + CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} go build -trimpath -ldflags="-s -w" -o dgdr-run-publisher ./cmd/dgdr-run-publisher && \ CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} go build -trimpath -ldflags="-s -w" -o dgd-apply-overrides ./cmd/dgd-apply-overrides # Emit a CycloneDX SBOM for the operator's Go module set so the licenses @@ -60,7 +61,7 @@ RUN CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} go build -o manager ./cm # v1.7.0 to keep the SBOM tool itself reproducible. # # `app -main ./cmd` covers the manager binary (./cmd/main.go = package -# main in ./cmd). crd-apply and dgd-apply-overrides are separate mains in +# main in ./cmd). crd-apply, dgdr-run-publisher and dgd-apply-overrides are separate mains in # the same module, so their deps are a subset of manager's — a single SBOM # is sufficient. Tried `mod` first but its license detection is weaker than # `app`'s — everything came back UNKNOWN. @@ -208,6 +209,7 @@ FROM nvcr.io/nvidia/distroless/go:v4.1.5 WORKDIR / COPY --from=builder /workspace/manager . COPY --from=builder /workspace/crd-apply . +COPY --from=builder /workspace/dgdr-run-publisher . COPY --from=builder /workspace/dgd-apply-overrides . COPY --from=builder /workspace/config/crd/bases/ /opt/dynamo-operator/crds/ COPY --from=licenses /legal /legal diff --git a/deploy/operator/Makefile b/deploy/operator/Makefile index 8224c328c29e..bcbcfe7a0942 100644 --- a/deploy/operator/Makefile +++ b/deploy/operator/Makefile @@ -4,7 +4,7 @@ IMG ?= controller:latest # ENVTEST_K8S_VERSION refers to the version of kubebuilder assets to be downloaded by envtest binary. ENVTEST_K8S_VERSION = 1.30.0 -ENVTEST_PACKAGES ?= ./internal/controller/... ./internal/webhook/validation ./cmd/crd-apply +ENVTEST_PACKAGES ?= ./internal/controller/... ./internal/webhook/validation ./cmd/crd-apply ./internal/dgdrrunpublisher ENVTEST_ASSETS_CMD = $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path CLUSTERTEST_PACKAGES ?= ./internal/testing/clusterenv ./internal/controller/... diff --git a/deploy/operator/api/v1beta2/dynamographdeploymentrequest_types.go b/deploy/operator/api/v1beta2/dynamographdeploymentrequest_types.go index 39887c2d6d17..37a22bd6c594 100644 --- a/deploy/operator/api/v1beta2/dynamographdeploymentrequest_types.go +++ b/deploy/operator/api/v1beta2/dynamographdeploymentrequest_types.go @@ -70,6 +70,7 @@ type HardwareSpec struct { // +kubebuilder:validation:MaxItems=2 // +kubebuilder:validation:items:Minimum=0 // +kubebuilder:validation:items:Maximum=1 +// +kubebuilder:validation:XValidation:rule="size(self) != 2 || self[0] < self[1]",message="a range needs min < max" type KVLoadRatio []float64 // StaticWorkloadSpec defines one of AISimulate's synthetic workload shapes. @@ -110,11 +111,10 @@ type StaticWorkloadSpec struct { // WorkloadSpec defines the synthetic traffic model used for every candidate evaluation. type WorkloadSpec struct { - // Static defines one of AISimulate's synthetic workload shapes. When omitted, - // it searches the full KV-load ratio range at ISL 4000 and OSL 1000. - // An explicit empty object opts out of this underlay. + // Static defines one of AISimulate's synthetic workload shapes. Its fields are not + // defaulted individually: the default belongs to spec.workload as a whole, so an + // explicit workload object keeps exactly the fields it names. // +optional - // +kubebuilder:default={isl: 4000, osl: 1000, kvLoadRatio: {0.0, 1.0}} Static *StaticWorkloadSpec `json:"static,omitempty"` } @@ -161,11 +161,10 @@ type ObjectiveSLASpec struct { type ObjectiveSpec struct { // Mode selects scalar optimization or a Pareto search. // +optional - // +kubebuilder:default=Optimize Mode ObjectiveMode `json:"mode,omitempty"` // Metric is the scalar target when mode is Optimize. - // When omitted in Optimize mode, admission defaults it to ThroughputPerGPU. + // When omitted in Optimize mode, admission defaults it to Throughput. // +optional Metric *ObjectiveMetric `json:"metric,omitempty"` @@ -176,24 +175,26 @@ type ObjectiveSpec struct { // +listType=set Metrics []ObjectiveMetric `json:"metrics,omitempty"` - // SLA defines per-request latency bounds used to calculate goodput. + // SLA defines per-request latency bounds used to calculate goodput. When omitted, no + // bounds apply. spec.aisimulate.parameters.goal.sla overrides the bounds it names. // +optional - // +kubebuilder:default={ttftMs: 2000, itlMs: 30} SLA *ObjectiveSLASpec `json:"sla,omitempty"` } // AISimulateSpec selects the AISimulate execution artifact and supplies native input. type AISimulateSpec struct { // Image selects a Dynamo-compatible AISimulate runner. When omitted, admission - // and execution defaults are resolved outside this API. + // persists the configured override or a version-matched Dynamo planner image. // +optional // +kubebuilder:validation:MinLength=1 Image string `json:"image,omitempty"` - // VersionOverride declares the AISimulate compatibility version when it - // cannot be inferred from the effective image. + // VersionOverride declares the AISimulate compatibility version as three numeric parts + // (for example 0.13.0), not a semver version: no v prefix, pre-release or build suffix. It is required + // when the image tag is not a version and wins over a version in the tag. // +optional // +kubebuilder:validation:MinLength=1 + // +kubebuilder:validation:Pattern=`^(0|[1-9][0-9]{0,3})\.(0|[1-9][0-9]{0,3})\.(0|[1-9][0-9]{0,3})$` VersionOverride string `json:"versionOverride,omitempty"` // Parameters contains native AISimulate input. Field names and nesting use @@ -210,7 +211,6 @@ type RecommendationSpec struct { // candidates are ordered best first; Pareto candidates have no scalar rank. // This field does not limit internal trials. // +optional - // +kubebuilder:default=1 // +kubebuilder:validation:Minimum=1 // +kubebuilder:validation:Maximum=100 MaxCandidates int32 `json:"maxCandidates,omitempty"` @@ -226,6 +226,7 @@ type NativeSearchInput struct { // Backends lists one or more inference backends searched by AISimulate and used by generated candidates. // +optional // +kubebuilder:validation:MinItems=1 + // +listType=set Backends []Backend `json:"backends,omitempty"` // Hardware bounds the accelerator configurations evaluated by the search. @@ -234,12 +235,19 @@ type NativeSearchInput struct { Hardware *HardwareSpec `json:"hardware,omitempty"` // Workload defines the synthetic traffic model used for every candidate evaluation. + // When omitted, admission supplies a static shape of ISL 1024, OSL 128 and concurrency 10; + // an explicit empty workload receives the same shape from admission. An explicit + // workload.static: {} opts out of that shape, leaving the workload to + // spec.aisimulate.parameters.workload. spec.aisimulate.parameters.workload overrides the + // fields it names. + // The default evaluates a fixed concurrency, so it creates neither sweep.max_trials nor a + // KV-load ratio search. // +optional Workload *WorkloadSpec `json:"workload,omitempty"` // Objective defines scalar optimization or a multi-objective Pareto search. - // When omitted, the schema supplies the default Optimize mode and SLA; - // admission supplies the conditional scalar metric. + // When omitted, admission supplies the object. Its mode defaults to Optimize, and admission + // supplies the conditional scalar metric. // +optional Objective *ObjectiveSpec `json:"objective,omitempty"` diff --git a/deploy/operator/api/v1beta2/dynamographdeploymentrun_conditions.go b/deploy/operator/api/v1beta2/dynamographdeploymentrun_conditions.go new file mode 100644 index 000000000000..fdc0f0dd1d50 --- /dev/null +++ b/deploy/operator/api/v1beta2/dynamographdeploymentrun_conditions.go @@ -0,0 +1,23 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package v1beta2 + +// DGDRRunConditionCompleted is the condition that makes a Run's outcome authoritative: it is True once +// the Run reached a terminal result. status.phase only describes how a completed Run ended, and +// consumers (such as the Request controller) decide on this condition, not on the phase. +const DGDRRunConditionCompleted = "Completed" diff --git a/deploy/operator/cmd/dgdr-run-publisher/main.go b/deploy/operator/cmd/dgdr-run-publisher/main.go new file mode 100644 index 000000000000..05c5c8f1cbd8 --- /dev/null +++ b/deploy/operator/cmd/dgdr-run-publisher/main.go @@ -0,0 +1,73 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Command dgdr-run-publisher is the sidecar of a DGDR(v2) run Job. It is a +// controller-runtime controller for one DynamoGraphDeploymentRun: it observes the +// Sweeper's pod-local snapshot file and reconciles it into DynamoGraphDeploymentCandidates +// and DynamoGraphDeploymentRun.status. See internal/dgdrrunpublisher. +package main + +import ( + "context" + "flag" + "fmt" + "os" + "os/signal" + "syscall" + "time" + + "github.com/ai-dynamo/dynamo/deploy/operator/internal/dgdrrunpublisher" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/log/zap" +) + +func main() { + os.Exit(run()) +} + +func run() int { + var cfg dgdrrunpublisher.Config + logOptions := zap.Options{} + logOptions.BindFlags(flag.CommandLine) + flag.StringVar(&cfg.SnapshotDir, "snapshot-dir", "", "directory shared with the Sweeper container") + flag.StringVar(&cfg.RunName, "run-name", "", "name of the DynamoGraphDeploymentRun") + flag.StringVar(&cfg.RunUID, "run-uid", "", "UID of the DynamoGraphDeploymentRun this pod serves") + flag.StringVar(&cfg.SweeperContainer, "sweeper-container", "sweeper", "name of the Sweeper container in this pod") + flag.DurationVar(&cfg.MinPollInterval, "poll-interval", time.Second, "snapshot check interval after a change") + flag.DurationVar( + &cfg.MaxPollInterval, "max-poll-interval", 5*time.Second, + "longest snapshot check interval while the snapshot is unchanged", + ) + flag.Parse() + ctrl.SetLogger(zap.New(zap.UseFlagOptions(&logOptions))) + + cfg.Namespace, cfg.PodName = os.Getenv("POD_NAMESPACE"), os.Getenv("POD_NAME") + if cfg.SnapshotDir == "" || cfg.RunName == "" || cfg.RunUID == "" || cfg.Namespace == "" || cfg.PodName == "" { + fmt.Fprintln(os.Stderr, "--snapshot-dir, --run-name, --run-uid and the POD_NAMESPACE/POD_NAME "+ + "environment variables are required") + return dgdrrunpublisher.ExitReconcileFailed + } + + ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGTERM, syscall.SIGINT) + defer stop() + + err := dgdrrunpublisher.Run(ctx, ctrl.GetConfigOrDie(), cfg) + if err != nil { + ctrl.Log.WithName("dgdr-run-publisher").Error(err, "publisher stopped") + } + return dgdrrunpublisher.ExitCode(err) +} diff --git a/deploy/operator/cmd/main.go b/deploy/operator/cmd/main.go index 6081b6b5acea..723ed748e6d3 100644 --- a/deploy/operator/cmd/main.go +++ b/deploy/operator/cmd/main.go @@ -69,12 +69,14 @@ import ( "github.com/ai-dynamo/dynamo/deploy/operator/internal/crdmigrator" lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx/scheduler/v1alpha1" "github.com/ai-dynamo/dynamo/deploy/operator/internal/features" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/gpu" "github.com/ai-dynamo/dynamo/deploy/operator/internal/namespace_scope" "github.com/ai-dynamo/dynamo/deploy/operator/internal/observability" "github.com/ai-dynamo/dynamo/deploy/operator/internal/podcache" "github.com/ai-dynamo/dynamo/deploy/operator/internal/rbac" "github.com/ai-dynamo/dynamo/deploy/operator/internal/secret" "github.com/ai-dynamo/dynamo/deploy/operator/internal/secrets" + webhookdefaulting "github.com/ai-dynamo/dynamo/deploy/operator/internal/webhook/defaulting" webhooksetup "github.com/ai-dynamo/dynamo/deploy/operator/internal/webhook/setup" grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" snapshotv1alpha1 "github.com/ai-dynamo/snapshot/api/v1alpha1" @@ -117,7 +119,6 @@ func initCRDSchemes() { utilruntime.Must(nvidiacomv1alpha1.AddToScheme(crdScheme)) utilruntime.Must(nvidiacomv1beta1.AddToScheme(crdScheme)) - utilruntime.Must(nvidiacomv1beta2.AddToScheme(crdScheme)) utilruntime.Must(lwsscheme.AddToScheme(crdScheme)) @@ -160,6 +161,8 @@ func main() { var operatorImage string var operatorImagePullPolicy string var dgdrDefaultImage string + var dgdrAISimulateDefaultImage string + var dgdrV1beta2Enabled bool flag.StringVar(&configFile, "config", "", "Path to operator configuration file (required)") flag.StringVar(&operatorVersion, "operator-version", "unknown", "Version of the operator (used in lease holder identity)") @@ -173,6 +176,11 @@ func main() { "Image pull policy for operator helper init containers") flag.StringVar(&dgdrDefaultImage, "dgdr-default-image", "", "Default DGDR profiler image, put into DGDR spec.image when unset; empty derives dynamo-planner:") + flag.StringVar(&dgdrAISimulateDefaultImage, "dgdr-aisimulate-default-image", "", + "AISimulate runner image, put into v1beta2 DGDR spec.aisimulate.image when unset; "+ + "empty derives dynamo-planner:") + flag.BoolVar(&dgdrV1beta2Enabled, "dgdr-v1beta2-enabled", false, + "Reconcile native v1beta2 DGDRs; requires the v1beta2 Request and Run CRDs to be installed") opts := zap.Options{ Development: true, } @@ -193,11 +201,13 @@ func main() { setupLog.Info("Operator configuration loaded successfully", "configFile", configFile) // Validate and normalize operator version to semver - if _, err := semver.NewVersion(operatorVersion); err != nil { + parsedOperatorVersion, err := semver.NewVersion(operatorVersion) + if err != nil { setupLog.Error(err, "operator-version is not valid semver", "provided", operatorVersion, "error", err.Error()) os.Exit(1) } + dynamoVersion := fmt.Sprintf("%d.%d.%d", parsedOperatorVersion.Major(), parsedOperatorVersion.Minor(), parsedOperatorVersion.Patch()) setupLog.Info("Operator version configured", "version", operatorVersion) pullPolicy := corev1.PullPolicy(operatorImagePullPolicy) @@ -498,14 +508,15 @@ func main() { if err := registerControllers( mgr, operatorCfg, runtimeConfig, dockerSecretRetriever, sshKeyManager, - operatorImage, pullPolicy, + operatorImage, pullPolicy, dynamoVersion, + dgdrV1beta2Enabled, ); err != nil { setupLog.Error(err, "failed to register controllers") os.Exit(1) } if err := registerWebhookHandlers( - mgr, operatorCfg, runtimeConfig, operatorVersion, dgdrDefaultImage, gates, + mgr, operatorCfg, runtimeConfig, operatorVersion, dgdrDefaultImage, dgdrAISimulateDefaultImage, gates, ); err != nil { setupLog.Error(err, "failed to register webhooks") os.Exit(1) @@ -548,6 +559,8 @@ func registerControllers( sshKeyManager *secret.SSHKeyManager, operatorImage string, operatorPullPolicy corev1.PullPolicy, + dynamoVersion string, + dgdrV1beta2Enabled bool, ) error { if operatorCfg.Namespace.Restricted == "" { // The cluster-wide manager cache covers all namespaces. ExcludedNamespaces only filters @@ -612,6 +625,19 @@ func registerControllers( }); err != nil { return err } + if dgdrV1beta2Enabled { + if err := controller.SetupDynamoGraphDeploymentRequestV1beta2(mgr, setupOptions); err != nil { + return err + } + if err := controller.SetupDynamoGraphDeploymentRunV1beta2(mgr, controller.DynamoGraphDeploymentRunSetupOptions{ + SetupOptions: setupOptions, + DynamoVersion: dynamoVersion, + OperatorImage: operatorImage, + OperatorImagePullPolicy: operatorPullPolicy, + }); err != nil { + return err + } + } if err := controller.SetupDynamoModel(mgr, controller.DynamoModelSetupOptions{ SetupOptions: setupOptions, }); err != nil { @@ -645,6 +671,7 @@ func registerWebhookHandlers( runtimeConfig *commonController.RuntimeConfig, operatorVersion string, dgdrDefaultImage string, + dgdrAISimulateDefaultImage string, gate features.Gate, ) error { var operatorPrincipal string @@ -656,10 +683,23 @@ func registerWebhookHandlers( } if err := webhooksetup.Setup(mgr, webhooksetup.Options{ - Config: operatorCfg, - RuntimeConfig: runtimeConfig, - OperatorVersion: operatorVersion, - DGDRDefaultImage: dgdrDefaultImage, + Config: operatorCfg, + RuntimeConfig: runtimeConfig, + OperatorVersion: operatorVersion, + DGDRV1beta1: webhookdefaulting.DGDRV1beta1Options{ + DefaultImage: dgdrDefaultImage, + }, + DGDRV1beta2: webhookdefaulting.DGDRV1beta2Options{ + AISimulateImage: dgdrAISimulateDefaultImage, + // The same discovery order as the DGDR reconciler: DCGM first, then node labels when + // the GPUDiscovery gate is on. The webhook keeps its own short-lived cache. + Hardware: &gpu.Discoverer{ + Reader: mgr.GetAPIReader(), + DCGM: gpu.NewGPUDiscovery(gpu.ScrapeMetricsEndpoint), + Cache: gpu.NewGPUDiscoveryCache(), + NodeLabels: func() bool { return gate.Enabled(features.GPUDiscovery) }, + }, + }, OperatorPrincipal: operatorPrincipal, Gate: gate, }); err != nil { diff --git a/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentrequests.yaml b/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentrequests.yaml index 404cc63ec777..411e08a3f8c4 100644 --- a/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentrequests.yaml +++ b/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentrequests.yaml @@ -9594,7 +9594,7 @@ spec: image: description: |- Image selects a Dynamo-compatible AISimulate runner. When omitted, admission - and execution defaults are resolved outside this API. + persists the configured override or a version-matched Dynamo planner image. minLength: 1 type: string parameters: @@ -9605,9 +9605,11 @@ spec: x-kubernetes-preserve-unknown-fields: true versionOverride: description: |- - VersionOverride declares the AISimulate compatibility version when it - cannot be inferred from the effective image. + VersionOverride declares the AISimulate compatibility version as three numeric parts + (for example 0.13.0), not a semver version: no v prefix, pre-release or build suffix. It is required + when the image tag is not a version and wins over a version in the tag. minLength: 1 + pattern: ^(0|[1-9][0-9]{0,3})\.(0|[1-9][0-9]{0,3})\.(0|[1-9][0-9]{0,3})$ type: string type: object backends: @@ -9621,6 +9623,7 @@ spec: type: string minItems: 1 type: array + x-kubernetes-list-type: set hardware: description: |- Hardware bounds the accelerator configurations evaluated by the search. @@ -9659,13 +9662,13 @@ spec: objective: description: |- Objective defines scalar optimization or a multi-objective Pareto search. - When omitted, the schema supplies the default Optimize mode and SLA; - admission supplies the conditional scalar metric. + When omitted, admission supplies the object. Its mode defaults to Optimize, and admission + supplies the conditional scalar metric. properties: metric: description: |- Metric is the scalar target when mode is Optimize. - When omitted in Optimize mode, admission defaults it to ThroughputPerGPU. + When omitted in Optimize mode, admission defaults it to Throughput. enum: - Throughput - ThroughputPerGPU @@ -9693,17 +9696,15 @@ spec: type: array x-kubernetes-list-type: set mode: - default: Optimize description: Mode selects scalar optimization or a Pareto search. enum: - Optimize - Pareto type: string sla: - default: - itlMs: 30 - ttftMs: 2000 - description: SLA defines per-request latency bounds used to calculate goodput. + description: |- + SLA defines per-request latency bounds used to calculate goodput. When omitted, no + bounds apply. spec.aisimulate.parameters.goal.sla overrides the bounds it names. properties: e2eMs: description: E2EMs is an alternative end-to-end bound in milliseconds. @@ -9736,7 +9737,6 @@ spec: description: Recommendation controls bounded projection into DGDC resources. properties: maxCandidates: - default: 1 description: |- MaxCandidates is the maximum number of DGDCs owned by one run. Scalar candidates are ordered best first; Pareto candidates have no scalar rank. @@ -9755,19 +9755,21 @@ spec: type: object x-kubernetes-preserve-unknown-fields: true workload: - description: Workload defines the synthetic traffic model used for every candidate evaluation. + description: |- + Workload defines the synthetic traffic model used for every candidate evaluation. + When omitted, admission supplies a static shape of ISL 1024, OSL 128 and concurrency 10; + an explicit empty workload receives the same shape from admission. An explicit + workload.static: {} opts out of that shape, leaving the workload to + spec.aisimulate.parameters.workload. spec.aisimulate.parameters.workload overrides the + fields it names. + The default evaluates a fixed concurrency, so it creates neither sweep.max_trials nor a + KV-load ratio search. properties: static: - default: - isl: 4000 - kvLoadRatio: - - 0 - - 1 - osl: 1000 description: |- - Static defines one of AISimulate's synthetic workload shapes. When omitted, - it searches the full KV-load ratio range at ISL 4000 and OSL 1000. - An explicit empty object opts out of this underlay. + Static defines one of AISimulate's synthetic workload shapes. Its fields are not + defaulted individually: the default belongs to spec.workload as a whole, so an + explicit workload object keeps exactly the fields it names. properties: concurrency: description: Concurrency is a fixed closed-loop in-flight request cap. @@ -9790,6 +9792,9 @@ spec: maxItems: 2 minItems: 1 type: array + x-kubernetes-validations: + - message: a range needs min < max + rule: size(self) != 2 || self[0] < self[1] numRequestRatio: description: NumRequestRatio sets request count relative to the selected load. exclusiveMinimum: true diff --git a/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentruns.yaml b/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentruns.yaml index 70a8e6f5e736..bb7ed65cfa81 100644 --- a/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentruns.yaml +++ b/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentruns.yaml @@ -66,7 +66,7 @@ spec: image: description: |- Image selects a Dynamo-compatible AISimulate runner. When omitted, admission - and execution defaults are resolved outside this API. + persists the configured override or a version-matched Dynamo planner image. minLength: 1 type: string parameters: @@ -77,9 +77,11 @@ spec: x-kubernetes-preserve-unknown-fields: true versionOverride: description: |- - VersionOverride declares the AISimulate compatibility version when it - cannot be inferred from the effective image. + VersionOverride declares the AISimulate compatibility version as three numeric parts + (for example 0.13.0), not a semver version: no v prefix, pre-release or build suffix. It is required + when the image tag is not a version and wins over a version in the tag. minLength: 1 + pattern: ^(0|[1-9][0-9]{0,3})\.(0|[1-9][0-9]{0,3})\.(0|[1-9][0-9]{0,3})$ type: string type: object backends: @@ -93,6 +95,7 @@ spec: type: string minItems: 1 type: array + x-kubernetes-list-type: set hardware: description: |- Hardware bounds the accelerator configurations evaluated by the search. @@ -131,13 +134,13 @@ spec: objective: description: |- Objective defines scalar optimization or a multi-objective Pareto search. - When omitted, the schema supplies the default Optimize mode and SLA; - admission supplies the conditional scalar metric. + When omitted, admission supplies the object. Its mode defaults to Optimize, and admission + supplies the conditional scalar metric. properties: metric: description: |- Metric is the scalar target when mode is Optimize. - When omitted in Optimize mode, admission defaults it to ThroughputPerGPU. + When omitted in Optimize mode, admission defaults it to Throughput. enum: - Throughput - ThroughputPerGPU @@ -165,17 +168,15 @@ spec: type: array x-kubernetes-list-type: set mode: - default: Optimize description: Mode selects scalar optimization or a Pareto search. enum: - Optimize - Pareto type: string sla: - default: - itlMs: 30 - ttftMs: 2000 - description: SLA defines per-request latency bounds used to calculate goodput. + description: |- + SLA defines per-request latency bounds used to calculate goodput. When omitted, no + bounds apply. spec.aisimulate.parameters.goal.sla overrides the bounds it names. properties: e2eMs: description: E2EMs is an alternative end-to-end bound in milliseconds. @@ -208,7 +209,6 @@ spec: description: Recommendation controls bounded projection into DGDC resources. properties: maxCandidates: - default: 1 description: |- MaxCandidates is the maximum number of DGDCs owned by one run. Scalar candidates are ordered best first; Pareto candidates have no scalar rank. @@ -219,19 +219,21 @@ spec: type: integer type: object workload: - description: Workload defines the synthetic traffic model used for every candidate evaluation. + description: |- + Workload defines the synthetic traffic model used for every candidate evaluation. + When omitted, admission supplies a static shape of ISL 1024, OSL 128 and concurrency 10; + an explicit empty workload receives the same shape from admission. An explicit + workload.static: {} opts out of that shape, leaving the workload to + spec.aisimulate.parameters.workload. spec.aisimulate.parameters.workload overrides the + fields it names. + The default evaluates a fixed concurrency, so it creates neither sweep.max_trials nor a + KV-load ratio search. properties: static: - default: - isl: 4000 - kvLoadRatio: - - 0 - - 1 - osl: 1000 description: |- - Static defines one of AISimulate's synthetic workload shapes. When omitted, - it searches the full KV-load ratio range at ISL 4000 and OSL 1000. - An explicit empty object opts out of this underlay. + Static defines one of AISimulate's synthetic workload shapes. Its fields are not + defaulted individually: the default belongs to spec.workload as a whole, so an + explicit workload object keeps exactly the fields it names. properties: concurrency: description: Concurrency is a fixed closed-loop in-flight request cap. @@ -254,6 +256,9 @@ spec: maxItems: 2 minItems: 1 type: array + x-kubernetes-validations: + - message: a range needs min < max + rule: size(self) != 2 || self[0] < self[1] numRequestRatio: description: NumRequestRatio sets request count relative to the selected load. exclusiveMinimum: true diff --git a/deploy/operator/internal/controller/aisimulate/defaults.go b/deploy/operator/internal/controller/aisimulate/defaults.go new file mode 100644 index 000000000000..8aecb2d8d569 --- /dev/null +++ b/deploy/operator/internal/controller/aisimulate/defaults.go @@ -0,0 +1,66 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package aisimulate + +import "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + +// The derived defaults. No API field carries them: each depends on the typed underlay, so +// no single typed field can supply it. +const ( + // DefaultRequestCount is workload.request_count when the typed underlay does not + // choose num_request_ratio. + DefaultRequestCount int64 = 1000 + + // DefaultMaxTrials bounds a KV-load range search that has no explicit sweep.max_trials. + // It matches the trial budget of AISimulate's unified CLI (optimizer.max_trials). + DefaultMaxTrials int64 = 320 +) + +// applyDefaults adds the derived defaults to the typed underlay, before the native +// parameters are applied. spec.aisimulate.parameters is the final overlay: it overrides or +// deletes any derived value, and nothing is derived in response to what it introduces. +// A native change that needs a matching change elsewhere (a request_count next to +// num_request_ratio, a strict_sla next to a new SLA bound) must make it itself. +func applyDefaults(underlay map[string]any) error { + b := builder{root: underlay} + + if has(underlay, workloadPath...) { + if !has(underlay, workloadRequestCount...) && !has(underlay, workloadNumRequestRatio...) { + b.set(float64(DefaultRequestCount), workloadRequestCount) + } + kvLoadRatio, _, _ := unstructured.NestedFieldNoCopy(underlay, workloadKVLoadRatio...) + if _, isRange := kvLoadRatio.([]any); isRange && !has(underlay, sweepMaxTrials...) { + b.set(float64(DefaultMaxTrials), sweepMaxTrials) + } + } + + if hasSLABound(underlay) && !has(underlay, goalStrictSLA...) { + b.set(true, goalStrictSLA) + } + return b.err +} + +// hasSLABound reports whether the input bounds at least one latency. +func hasSLABound(input map[string]any) bool { + for _, path := range [][]string{goalSLATTFTMs, goalSLAITLMs, goalSLAE2EMs} { + if bound, _, _ := unstructured.NestedFieldNoCopy(input, path...); bound != nil { + return true + } + } + return false +} diff --git a/deploy/operator/internal/controller/aisimulate/doc.go b/deploy/operator/internal/controller/aisimulate/doc.go new file mode 100644 index 000000000000..a15cb22952e7 --- /dev/null +++ b/deploy/operator/internal/controller/aisimulate/doc.go @@ -0,0 +1,38 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package aisimulate lowers a v1beta2 DynamoGraphDeploymentRequest spec into the native +// AISimulate input that a Run executes. +// +// Lowering sits below the Request reconciler. Admission does not call it: spec.aisimulate.parameters +// is opaque there, and the validation of the lowered input belongs to the schema of the AISimulate +// release the Request selects. The reconciler lowers the Request once and freezes the output into the +// Run, which executes that object as it is. +// +// Lower works in three steps. The typed spec (model, backends, hardware, workload, objective) +// is first translated into the native input, the "underlay". The derived defaults that no API +// field carries are then added to it: the request count, a bounded trial budget for a +// KV-load range search, and goal.strict_sla when an SLA bound is set. Last, +// spec.aisimulate.parameters is applied on top as a JSON Merge Patch (RFC 7386): native +// objects merge recursively, arrays and scalars replace the value, and null deletes it. The +// parameters are authoritative: they win on every path they name, including the derived +// defaults, and are applied once, last. +// +// The document stays unstructured (map[string]any) because its schema belongs to the +// AISimulate release the Request selects, not to this package. Reads and writes go +// through the helpers of k8s.io/apimachinery/pkg/apis/meta/v1/unstructured. +package aisimulate diff --git a/deploy/operator/internal/controller/aisimulate/hardware_named_test.go b/deploy/operator/internal/controller/aisimulate/hardware_named_test.go new file mode 100644 index 000000000000..fdc1e2167158 --- /dev/null +++ b/deploy/operator/internal/controller/aisimulate/hardware_named_test.go @@ -0,0 +1,43 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package aisimulate + +import "testing" + +func TestHardwareNamed(t *testing.T) { + tests := []struct { + name string + input map[string]any + want bool + }{ + {"both named", map[string]any{"search_space": map[string]any{"hardware_sku": "h200_sxm", "gpu_budget": int64(8)}}, true}, + {"no search space", map[string]any{}, false}, + {"budget missing", map[string]any{"search_space": map[string]any{"hardware_sku": "h200_sxm"}}, false}, + {"sku missing", map[string]any{"search_space": map[string]any{"gpu_budget": int64(8)}}, false}, + {"null sku", map[string]any{"search_space": map[string]any{"hardware_sku": nil, "gpu_budget": int64(8)}}, false}, + {"blank sku", map[string]any{"search_space": map[string]any{"hardware_sku": "", "gpu_budget": int64(8)}}, false}, + {"null budget", map[string]any{"search_space": map[string]any{"hardware_sku": "h200_sxm", "gpu_budget": nil}}, false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := HardwareNamed(tt.input); got != tt.want { + t.Fatalf("HardwareNamed() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/deploy/operator/internal/controller/aisimulate/image.go b/deploy/operator/internal/controller/aisimulate/image.go new file mode 100644 index 000000000000..91afdcb4496f --- /dev/null +++ b/deploy/operator/internal/controller/aisimulate/image.go @@ -0,0 +1,40 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package aisimulate + +import ( + "errors" + "fmt" + + "github.com/ai-dynamo/dynamo/deploy/operator/internal/versionoverride" +) + +// ResolveVersion returns the AISimulate compatibility version of a Request: the explicit +// override when set, otherwise the version in the image tag. An image whose tag is not a +// version, or a missing image, is an error; the caller reports it on the field that fixes it. +// The parsing is shared with the runtime version of the other Request kinds. +func ResolveVersion(image, versionOverride string) (string, error) { + if image == "" && versionOverride == "" { + return "", errors.New("no AISimulate image") + } + version, err := versionoverride.Resolve(image, versionOverride) + if err != nil { + return "", fmt.Errorf("resolve AISimulate version: %w", err) + } + return version.String(), nil +} diff --git a/deploy/operator/internal/controller/aisimulate/image_test.go b/deploy/operator/internal/controller/aisimulate/image_test.go new file mode 100644 index 000000000000..064c3b80b2a8 --- /dev/null +++ b/deploy/operator/internal/controller/aisimulate/image_test.go @@ -0,0 +1,45 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package aisimulate + +import "testing" + +func TestResolveVersion(t *testing.T) { + cases := []struct { + name, image, override, want string + wantErr bool + }{ + {name: "version from the tag", image: "registry.example.com/aisimulate:1.2.3", want: "1.2.3"}, + {name: "v-prefixed tag", image: "registry.example.com/aisimulate:v1.2.3", want: "1.2.3"}, + {name: "override wins over the tag", image: "registry.example.com/aisimulate:1.2.3", override: "2.0.0", want: "2.0.0"}, + {name: "override rescues a custom tag", image: "registry.example.com/aisimulate:custom", override: "1.4.0", want: "1.4.0"}, + {name: "custom tag needs an override", image: "registry.example.com/aisimulate:custom", wantErr: true}, + {name: "untagged image needs an override", image: "registry.example.com/aisimulate", wantErr: true}, + {name: "registry port is not a tag", image: "registry.example.com:5000/aisimulate", wantErr: true}, + {name: "no image", wantErr: true}, + {name: "override without an image", override: "1.0.0", want: "1.0.0"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got, err := ResolveVersion(tc.image, tc.override) + if (err != nil) != tc.wantErr || got != tc.want { + t.Fatalf("ResolveVersion(%q, %q) = %q, %v; want %q (error %v)", tc.image, tc.override, got, err, tc.want, tc.wantErr) + } + }) + } +} diff --git a/deploy/operator/internal/controller/aisimulate/input.go b/deploy/operator/internal/controller/aisimulate/input.go new file mode 100644 index 000000000000..4eaae415c8ad --- /dev/null +++ b/deploy/operator/internal/controller/aisimulate/input.go @@ -0,0 +1,268 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package aisimulate + +import ( + "encoding/json" + "fmt" + "strings" + "unicode" + + v1beta2 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta2" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" +) + +// Lower returns the effective native input: the typed underlay, merged with +// spec.aisimulate.parameters, plus the derived defaults. The spec must already carry its +// structural defaults; lowering repeats none of them. The only error is malformed +// parameters (not a JSON object); semantic problems are the caller's to validate. +func Lower(spec *v1beta2.DynamoGraphDeploymentRequestSpec) (map[string]any, error) { + parameters, err := Parameters(spec) + if err != nil { + return nil, err + } + underlay, err := Underlay(spec) + if err != nil { + return nil, err + } + if underlay, err = normalize(underlay); err != nil { + return nil, err + } + if err := applyDefaults(underlay); err != nil { + return nil, err + } + return MergeParameters(underlay, parameters), nil +} + +// MergeParameters applies parameters as an RFC 7386 merge patch over underlay. Native +// AISimulate parameters are always the final overlay: they may override or delete values +// supplied by a typed API or by controller-owned execution defaults. +func MergeParameters(underlay, parameters map[string]any) map[string]any { + input, _ := mergePatch(underlay, parameters).(map[string]any) + if input == nil { + return map[string]any{} + } + return input +} + +// Parameters returns spec.aisimulate.parameters as a JSON object, or nil when absent. +func Parameters(spec *v1beta2.DynamoGraphDeploymentRequestSpec) (map[string]any, error) { + if spec.AISimulate == nil || spec.AISimulate.Parameters == nil { + return nil, nil + } + return DecodeParameters(spec.AISimulate.Parameters) +} + +// DecodeParameters returns a native AISimulate parameters object without applying typed +// API fields or defaults. A Run controller uses this after the Request controller has frozen +// the complete effective input into spec.aisimulate.parameters. +func DecodeParameters(parameters *runtime.RawExtension) (map[string]any, error) { + if parameters == nil { + return nil, nil + } + raw := parameters.Raw + if len(raw) == 0 && parameters.Object != nil { + var err error + if raw, err = json.Marshal(parameters.Object); err != nil { + return nil, fmt.Errorf("spec.aisimulate.parameters: %w", err) + } + } + if len(raw) == 0 { + return nil, nil + } + var value any + if err := json.Unmarshal(raw, &value); err != nil { + return nil, fmt.Errorf("spec.aisimulate.parameters: %w", err) + } + switch typed := value.(type) { + case nil: + return nil, nil + case map[string]any: + return typed, nil + default: + return nil, fmt.Errorf("spec.aisimulate.parameters: must be a JSON object, got %T", value) + } +} + +// Underlay translates the typed spec into native input. Typed fields that are unset +// produce no key, so the native defaults and the parameters decide them. +func Underlay(spec *v1beta2.DynamoGraphDeploymentRequestSpec) (map[string]any, error) { + b := builder{root: map[string]any{}} + + if spec.Model != nil { + b.set(spec.Model.Name, searchSpaceModelName) + } + if len(spec.Backends) > 0 { + backends := make([]string, len(spec.Backends)) + for i, backend := range spec.Backends { + backends[i] = string(backend) + } + b.setStrings(backends, searchSpaceBackend) + } + if spec.Hardware != nil && spec.Hardware.GPU != nil { + if sku := spec.Hardware.GPU.SKU; sku != nil && *sku != "" { + b.set(*sku, searchSpaceHardwareSKU) + } + if budget := spec.Hardware.GPU.Budget; budget != nil { + b.set(int64(*budget), searchSpaceGPUBudget) + } + } + + if spec.Workload != nil && spec.Workload.Static != nil { + static := spec.Workload.Static + if static.ISL != nil { + b.set(int64(*static.ISL), workloadISL) + } + if static.OSL != nil { + b.set(int64(*static.OSL), workloadOSL) + } + if static.NumRequestRatio != nil { + b.set(*static.NumRequestRatio, workloadNumRequestRatio) + } + if static.RequestRate != nil { + b.set(*static.RequestRate, workloadRequestRate) + } + if static.Concurrency != nil { + b.set(int64(*static.Concurrency), workloadConcurrency) + } + switch len(static.KVLoadRatio) { + case 0: + case 1: + b.set(static.KVLoadRatio[0], workloadKVLoadRatio) + default: + ratios := make([]any, len(static.KVLoadRatio)) + for i, ratio := range static.KVLoadRatio { + ratios[i] = ratio + } + b.set(ratios, workloadKVLoadRatio) + } + } + + if objective := spec.Objective; objective != nil { + if objective.Mode == v1beta2.ObjectiveModePareto { + b.set(TargetPareto, goalTarget) + if len(objective.Metrics) > 0 { + b.setStrings(nativeMetrics(objective.Metrics), goalParetoObjectives) + } + } else if objective.Metric != nil { + b.set(nativeMetric(*objective.Metric), goalTarget) + } + if sla := objective.SLA; sla != nil { + if sla.TTFTMs != nil { + b.set(*sla.TTFTMs, goalSLATTFTMs) + } + if sla.ITLMs != nil { + b.set(*sla.ITLMs, goalSLAITLMs) + } + if sla.E2EMs != nil { + b.set(*sla.E2EMs, goalSLAE2EMs) + } + } + } + return b.root, b.err +} + +// builder writes native input values, remembering the first failure so the translation reads +// as a straight list of assignments. Every path is written under a map the caller created +// and every value is a JSON scalar or a list of them, so a failure means a bad path. +type builder struct { + root map[string]any + err error +} + +func (b *builder) set(value any, path []string) { + if b.err != nil { + return + } + if err := unstructured.SetNestedField(b.root, value, path...); err != nil { + b.err = fmt.Errorf("set %s: %w", strings.Join(path, "."), err) + } +} + +func (b *builder) setStrings(values []string, path []string) { + if b.err != nil { + return + } + if err := unstructured.SetNestedStringSlice(b.root, values, path...); err != nil { + b.err = fmt.Errorf("set %s: %w", strings.Join(path, "."), err) + } +} + +// nativeMetric maps an API metric to AISimulate's goal.target by converting its CamelCase +// name to snake_case: ThroughputPerGPU becomes throughput_per_gpu and E2ELatency becomes +// e2e_latency. A run of capitals stays together, a digit does not start a word, and the last +// capital of a run starts the next word when a lowercase letter follows it. +func nativeMetric(metric v1beta2.ObjectiveMetric) string { + name := []rune(string(metric)) + var out []rune + for i, r := range name { + if i > 0 && unicode.IsUpper(r) { + prev := name[i-1] + nextLower := i+1 < len(name) && unicode.IsLower(name[i+1]) + if unicode.IsLower(prev) || (unicode.IsUpper(prev) && nextLower) { + out = append(out, '_') + } + } + out = append(out, unicode.ToLower(r)) + } + return string(out) +} + +func nativeMetrics(metrics []v1beta2.ObjectiveMetric) []string { + out := make([]string, len(metrics)) + for i, metric := range metrics { + out[i] = nativeMetric(metric) + } + return out +} + +// normalize round-trips through JSON so every number is a float64, the type the +// parameters decode to. Merge and validation then see one representation. +func normalize(value map[string]any) (map[string]any, error) { + raw, err := json.Marshal(value) + if err != nil { + return nil, err + } + var out map[string]any + if err := json.Unmarshal(raw, &out); err != nil { + return nil, err + } + return out, nil +} + +// mergePatch applies patch to target with RFC 7386 semantics. It returns a new value for +// the merged parts and does not retain the patch's maps for null members. +func mergePatch(target, patch any) any { + patchMap, ok := patch.(map[string]any) + if !ok { + return patch + } + targetMap, ok := target.(map[string]any) + if !ok { + targetMap = map[string]any{} + } + for key, value := range patchMap { + if value == nil { + delete(targetMap, key) + continue + } + targetMap[key] = mergePatch(targetMap[key], value) + } + return targetMap +} diff --git a/deploy/operator/internal/controller/aisimulate/input_test.go b/deploy/operator/internal/controller/aisimulate/input_test.go new file mode 100644 index 000000000000..dfeff5a150a0 --- /dev/null +++ b/deploy/operator/internal/controller/aisimulate/input_test.go @@ -0,0 +1,433 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package aisimulate + +import ( + "encoding/json" + "strings" + "testing" + + v1beta2 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta2" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" +) + +func ptrTo[T any](v T) *T { return &v } + +// defaultedSpec is a model-only Request as it looks after the CRD's structural defaults and +// admission: the fixed-concurrency workload, the scalar metric and discovered hardware. It +// has no SLA bounds and no KV-load range; withSLA and withKVRange add them. +func defaultedSpec() *v1beta2.DynamoGraphDeploymentRequestSpec { + metric := v1beta2.ObjectiveMetric("Throughput") + return &v1beta2.DynamoGraphDeploymentRequestSpec{ + Model: &v1beta2.ModelSpec{Name: "meta-llama/Llama-3.1-8B-Instruct"}, + Backends: []v1beta2.Backend{v1beta2.BackendVLLM, v1beta2.BackendSGLang, v1beta2.BackendTRTLLM}, + Hardware: &v1beta2.HardwareSpec{GPU: &v1beta2.GPUHardwareSpec{SKU: ptrTo("h200_sxm"), Budget: ptrTo(int32(8))}}, + Workload: &v1beta2.WorkloadSpec{Static: &v1beta2.StaticWorkloadSpec{ + ISL: ptrTo(int32(1024)), OSL: ptrTo(int32(128)), Concurrency: ptrTo(int32(10)), + }}, + Objective: &v1beta2.ObjectiveSpec{Mode: v1beta2.ObjectiveModeOptimize, Metric: &metric}, + Recommendation: &v1beta2.RecommendationSpec{MaxCandidates: 1}, + } +} + +// withSLA adds the latency bounds the earlier defaults used to supply. +func withSLA(spec *v1beta2.DynamoGraphDeploymentRequestSpec) *v1beta2.DynamoGraphDeploymentRequestSpec { + spec.Objective.SLA = &v1beta2.ObjectiveSLASpec{TTFTMs: ptrTo(2000.0), ITLMs: ptrTo(30.0)} + return spec +} + +// withKVRange replaces the fixed concurrency with a KV-load range search. +func withKVRange(spec *v1beta2.DynamoGraphDeploymentRequestSpec) *v1beta2.DynamoGraphDeploymentRequestSpec { + spec.Workload.Static = &v1beta2.StaticWorkloadSpec{ + ISL: ptrTo(int32(4000)), OSL: ptrTo(int32(1000)), KVLoadRatio: v1beta2.KVLoadRatio{0, 1}, + } + return spec +} + +func withParameters(spec *v1beta2.DynamoGraphDeploymentRequestSpec, parameters string) *v1beta2.DynamoGraphDeploymentRequestSpec { + spec.AISimulate = &v1beta2.AISimulateSpec{Parameters: &runtime.RawExtension{Raw: []byte(parameters)}} + return spec +} + +func lower(t *testing.T, spec *v1beta2.DynamoGraphDeploymentRequestSpec) map[string]any { + t.Helper() + input, err := Lower(spec) + if err != nil { + t.Fatalf("Lower: %v", err) + } + return input +} + +func encode(t *testing.T, input map[string]any) string { + t.Helper() + raw, err := json.Marshal(input) + if err != nil { + t.Fatal(err) + } + return string(raw) +} + +func underlay(t *testing.T, spec *v1beta2.DynamoGraphDeploymentRequestSpec) map[string]any { + t.Helper() + input, err := Underlay(spec) + if err != nil { + t.Fatalf("Underlay: %v", err) + } + return input +} + +func nested(t *testing.T, input map[string]any, path ...string) any { + t.Helper() + value, _, err := unstructured.NestedFieldNoCopy(input, path...) + if err != nil { + t.Fatal(err) + } + return value +} + +func TestLowerModelOnlyRequest(t *testing.T) { + want := `{"goal":{"target":"throughput"},` + + `"search_space":{"backend":["vllm","sglang","trtllm"],"gpu_budget":8,"hardware_sku":"h200_sxm","model_name":"meta-llama/Llama-3.1-8B-Instruct"},` + + `"workload":{"concurrency":10,"isl":1024,"osl":128,"request_count":1000}}` + if got := encode(t, lower(t, defaultedSpec())); got != want { + t.Fatalf("effective input\n got: %s\nwant: %s", got, want) + } +} + +func TestLowerIsDeterministicAndDoesNotMutateTheSpec(t *testing.T) { + const parameters = `{"sweep":{"max_trials":100},"workload":{"request_count":5}}` + spec := withParameters(defaultedSpec(), parameters) + first, second := encode(t, lower(t, spec)), encode(t, lower(t, spec)) + if first != second { + t.Fatalf("not deterministic:\n%s\n%s", first, second) + } + if string(spec.AISimulate.Parameters.Raw) != parameters { + t.Fatalf("parameters mutated: %s", spec.AISimulate.Parameters.Raw) + } +} + +func TestLowerExplicitRequest(t *testing.T) { + metric := v1beta2.ObjectiveMetric("Throughput") + spec := withSLA(defaultedSpec()) + spec.Backends = []v1beta2.Backend{v1beta2.BackendVLLM, v1beta2.BackendSGLang} + spec.Workload.Static = &v1beta2.StaticWorkloadSpec{ISL: ptrTo(int32(4000)), OSL: ptrTo(int32(1000)), RequestRate: ptrTo(10.0)} + spec.Objective.Metric = &metric + spec = withParameters(spec, `{"workload":{"request_count":1000},"sweep":{"max_trials":100}}`) + + want := `{"goal":{"sla":{"itl_ms":30,"ttft_ms":2000},"strict_sla":true,"target":"throughput"},` + + `"search_space":{"backend":["vllm","sglang"],"gpu_budget":8,"hardware_sku":"h200_sxm","model_name":"meta-llama/Llama-3.1-8B-Instruct"},` + + `"sweep":{"max_trials":100},` + + `"workload":{"isl":4000,"osl":1000,"request_count":1000,"request_rate":10}}` + if got := encode(t, lower(t, spec)); got != want { + t.Fatalf("effective input\n got: %s\nwant: %s", got, want) + } +} + +func TestMergePatchSemantics(t *testing.T) { + cases := []struct{ name, target, patch, want string }{ + {"scalar replaces", `{"a":1}`, `{"a":2}`, `{"a":2}`}, + {"object merges recursively", `{"a":{"b":1,"c":2}}`, `{"a":{"b":3}}`, `{"a":{"b":3,"c":2}}`}, + {"null deletes", `{"a":1,"b":2}`, `{"a":null}`, `{"b":2}`}, + {"array replaces", `{"a":[1,2,3]}`, `{"a":[4]}`, `{"a":[4]}`}, + {"object replaces scalar and drops nulls", `{"a":1}`, `{"a":{"b":null,"c":1}}`, `{"a":{"c":1}}`}, + {"null on absent key is a no-op", `{"a":1}`, `{"z":null}`, `{"a":1}`}, + {"empty patch", `{"a":1}`, `{}`, `{"a":1}`}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + var target, patch any + if err := json.Unmarshal([]byte(tc.target), &target); err != nil { + t.Fatal(err) + } + if err := json.Unmarshal([]byte(tc.patch), &patch); err != nil { + t.Fatal(err) + } + got, _ := json.Marshal(mergePatch(target, patch)) + if string(got) != tc.want { + t.Fatalf("got %s want %s", got, tc.want) + } + }) + } + t.Run("nil patch map", func(t *testing.T) { + target := map[string]any{"a": 1.0} + if got := mergePatch(target, map[string]any(nil)); got == nil { + t.Fatal("a nil patch must keep the target") + } + }) +} + +func TestNativeValuesWinOverTheTypedUnderlay(t *testing.T) { + t.Run("a native null deletes a typed field", func(t *testing.T) { + input := lower(t, withParameters(defaultedSpec(), `{"workload":{"isl":null}}`)) + if nested(t, input, "workload", "isl") != nil { + t.Fatalf("isl survived a native null: %s", encode(t, input)) + } + }) + t.Run("a native value overrides a typed field", func(t *testing.T) { + input := lower(t, withParameters(defaultedSpec(), `{"workload":{"isl":8000},"search_space":{"gpu_budget":16}}`)) + if nested(t, input, "workload", "isl") != 8000.0 || nested(t, input, "search_space", "gpu_budget") != 16.0 { + t.Fatalf("native values did not win: %s", encode(t, input)) + } + }) + t.Run("a native target overrides the typed metric", func(t *testing.T) { + input := lower(t, withParameters(defaultedSpec(), `{"goal":{"target":"ttft"}}`)) + if nested(t, input, "goal", "target") != "ttft" { + t.Fatalf("typed metric beat the native target: %s", encode(t, input)) + } + }) + t.Run("a native SLA bound overrides only the bound it names", func(t *testing.T) { + input := lower(t, withParameters(withSLA(defaultedSpec()), `{"goal":{"sla":{"ttft_ms":500}}}`)) + if nested(t, input, "goal", "sla", "ttft_ms") != 500.0 || nested(t, input, "goal", "sla", "itl_ms") != 30.0 { + t.Fatalf("unexpected SLA: %s", encode(t, input)) + } + }) +} + +func TestRequestCountDefault(t *testing.T) { + t.Run("derived from the typed underlay", func(t *testing.T) { + if nested(t, lower(t, defaultedSpec()), "workload", "request_count") != 1000.0 { + t.Fatal("request_count not derived") + } + }) + t.Run("a native ratio deletes the derived count itself", func(t *testing.T) { + input := lower(t, withParameters(defaultedSpec(), `{"workload":{"num_request_ratio":10,"request_count":null}}`)) + if nested(t, input, "workload", "request_count") != nil || nested(t, input, "workload", "num_request_ratio") != 10.0 { + t.Fatalf("unexpected workload: %s", encode(t, input)) + } + }) + t.Run("skipped when typed numRequestRatio is set", func(t *testing.T) { + spec := defaultedSpec() + spec.Workload.Static.NumRequestRatio = ptrTo(5.0) + if nested(t, lower(t, spec), "workload", "request_count") != nil { + t.Fatal("request_count defaulted next to typed numRequestRatio") + } + }) + t.Run("native null deletes the default", func(t *testing.T) { + input := lower(t, withParameters(defaultedSpec(), `{"workload":{"request_count":null}}`)) + if nested(t, input, "workload", "request_count") != nil { + t.Fatal("request_count re-added after a native null") + } + }) + t.Run("native deleting the typed ratio does not derive a count", func(t *testing.T) { + spec := defaultedSpec() + spec.Workload.Static.NumRequestRatio = ptrTo(5.0) + input := lower(t, withParameters(spec, `{"workload":{"num_request_ratio":null}}`)) + if nested(t, input, "workload", "request_count") != nil { + t.Fatalf("a count was derived after the native overlay: %s", encode(t, input)) + } + }) + t.Run("not added without a workload", func(t *testing.T) { + spec := defaultedSpec() + spec.Workload = &v1beta2.WorkloadSpec{} + if nested(t, lower(t, spec), "workload") != nil { + t.Fatal("a workload appeared from nothing") + } + }) +} + +func TestMaxTrialsDefault(t *testing.T) { + t.Run("added for a KV-load range", func(t *testing.T) { + if nested(t, lower(t, withKVRange(defaultedSpec())), "sweep", "max_trials") != 320.0 { + t.Fatal("no default max_trials for a range search") + } + }) + t.Run("not derived for the model-only fixed concurrency", func(t *testing.T) { + if input := lower(t, defaultedSpec()); nested(t, input, "sweep") != nil { + t.Fatalf("unexpected sweep: %s", encode(t, input)) + } + }) + t.Run("not added for a pinned load", func(t *testing.T) { + spec := withKVRange(defaultedSpec()) + spec.Workload.Static.KVLoadRatio = v1beta2.KVLoadRatio{0.5} + input := lower(t, spec) + if nested(t, input, "sweep") != nil { + t.Fatalf("unexpected sweep: %s", encode(t, input)) + } + if nested(t, input, "workload", "kv_load_ratio") != 0.5 { + t.Fatal("a one-value ratio should be a scalar") + } + }) + t.Run("native value or null wins", func(t *testing.T) { + if nested(t, lower(t, withParameters(withKVRange(defaultedSpec()), `{"sweep":{"max_trials":7}}`)), "sweep", "max_trials") != 7.0 { + t.Fatal("native max_trials ignored") + } + if nested(t, lower(t, withParameters(withKVRange(defaultedSpec()), `{"sweep":{"max_trials":null}}`)), "sweep", "max_trials") != nil { + t.Fatal("max_trials re-added after a native null") + } + }) +} + +func TestStrictSLADerivation(t *testing.T) { + t.Run("enabled when the typed SLA has a bound", func(t *testing.T) { + if nested(t, lower(t, withSLA(defaultedSpec())), "goal", "strict_sla") != true { + t.Fatal("strict_sla not derived") + } + }) + t.Run("model-only derives nothing", func(t *testing.T) { + input := lower(t, defaultedSpec()) + if nested(t, input, "goal", "strict_sla") != nil || nested(t, input, "goal", "sla") != nil { + t.Fatalf("model-only input has SLA keys: %s", encode(t, input)) + } + }) + t.Run("an empty SLA derives nothing", func(t *testing.T) { + spec := defaultedSpec() + spec.Objective.SLA = &v1beta2.ObjectiveSLASpec{} + input := lower(t, spec) + if nested(t, input, "goal", "strict_sla") != nil || nested(t, input, "goal", "sla") != nil { + t.Fatalf("an empty SLA produced SLA keys: %s", encode(t, input)) + } + }) + t.Run("a native bound alone does not derive it", func(t *testing.T) { + input := lower(t, withParameters(defaultedSpec(), `{"goal":{"sla":{"ttft_ms":500}}}`)) + if nested(t, input, "goal", "sla", "ttft_ms") != 500.0 || nested(t, input, "goal", "strict_sla") != nil { + t.Fatalf("unexpected goal: %s", encode(t, input)) + } + }) + t.Run("native value is kept", func(t *testing.T) { + if nested(t, lower(t, withParameters(withSLA(defaultedSpec()), `{"goal":{"strict_sla":false}}`)), "goal", "strict_sla") != false { + t.Fatal("native strict_sla overridden") + } + }) + t.Run("is not revisited after the native overlay", func(t *testing.T) { + input := lower(t, withParameters(withSLA(defaultedSpec()), `{"goal":{"sla":{"ttft_ms":null,"itl_ms":null}}}`)) + if nested(t, input, "goal", "strict_sla") != true { + t.Fatalf("the derived strict_sla changed after native bounds were deleted: %s", encode(t, input)) + } + }) +} + +func TestParametersApplyOnceAfterDerivedDefaults(t *testing.T) { + t.Run("sweep null removes the complete derived sweep", func(t *testing.T) { + input := lower(t, withParameters(withKVRange(defaultedSpec()), `{"sweep":null}`)) + if nested(t, input, "sweep") != nil { + t.Fatalf("sweep survived a native null: %s", encode(t, input)) + } + }) + t.Run("max_trials null removes only max_trials", func(t *testing.T) { + input := lower(t, withParameters(withKVRange(defaultedSpec()), `{"sweep":{"max_trials":null,"seed":3}}`)) + if nested(t, input, "sweep", "max_trials") != nil || nested(t, input, "sweep", "seed") != 3.0 { + t.Fatalf("unexpected sweep: %s", encode(t, input)) + } + }) + t.Run("a native max_trials replaces the derived value", func(t *testing.T) { + input := lower(t, withParameters(withKVRange(defaultedSpec()), `{"sweep":{"max_trials":7}}`)) + if nested(t, input, "sweep", "max_trials") != 7.0 { + t.Fatalf("unexpected sweep: %s", encode(t, input)) + } + }) + t.Run("nothing is derived from values only parameters introduce", func(t *testing.T) { + input := lower(t, withParameters(defaultedSpec(), `{"workload":{"kv_load_ratio":[0,1]},"goal":{"sla":{"ttft_ms":500}}}`)) + if nested(t, input, "sweep") != nil || nested(t, input, "goal", "strict_sla") != nil { + t.Fatalf("a field was derived from native-only values: %s", encode(t, input)) + } + }) +} + +func TestUnderlayParetoObjective(t *testing.T) { + spec := defaultedSpec() + spec.Objective.Mode = v1beta2.ObjectiveModePareto + spec.Objective.Metric = nil + spec.Objective.Metrics = []v1beta2.ObjectiveMetric{"ThroughputPerGPU", "ThroughputPerUser", "Unknown"} + input := underlay(t, spec) + if nested(t, input, "goal", "target") != TargetPareto { + t.Fatalf("not a pareto goal: %v", input["goal"]) + } + objectives, _, _ := unstructured.NestedSlice(input, "goal", "pareto_objectives") + got, _ := json.Marshal(objectives) + if string(got) != `["throughput_per_gpu","throughput_per_user","unknown"]` { + t.Fatalf("pareto_objectives = %s", got) + } +} + +func TestUnderlayOmitsUnsetTypedFields(t *testing.T) { + spec := &v1beta2.DynamoGraphDeploymentRequestSpec{Model: &v1beta2.ModelSpec{Name: "m"}} + if got := encode(t, underlay(t, spec)); got != `{"search_space":{"model_name":"m"}}` { + t.Fatalf("got %s", got) + } + spec.Hardware = &v1beta2.HardwareSpec{GPU: &v1beta2.GPUHardwareSpec{SKU: ptrTo("")}} + if got := encode(t, underlay(t, spec)); got != `{"search_space":{"model_name":"m"}}` { + t.Fatalf("an empty SKU must not lower: %s", got) + } +} + +func TestUnderlayKVLoadRatioShapes(t *testing.T) { + spec := defaultedSpec() + spec.Workload.Static.KVLoadRatio = v1beta2.KVLoadRatio{0.25} + if nested(t, underlay(t, spec), "workload", "kv_load_ratio") != 0.25 { + t.Fatal("a single ratio lowers to a scalar") + } + spec.Workload.Static.KVLoadRatio = v1beta2.KVLoadRatio{0.1, 0.2, 0.3} + if ratios, _ := nested(t, underlay(t, spec), "workload", "kv_load_ratio").([]any); len(ratios) != 3 { + t.Fatal("longer lists lower unchanged so validation can report them") + } +} + +func TestParametersMustBeAnObject(t *testing.T) { + for _, parameters := range []string{`[1]`, `"x"`, `3`, `{`} { + if _, err := Lower(withParameters(defaultedSpec(), parameters)); err == nil { + t.Errorf("Lower accepted parameters %s", parameters) + } + } + for _, parameters := range []string{``, `null`, `{}`} { + if _, err := Lower(withParameters(defaultedSpec(), parameters)); err != nil { + t.Errorf("Lower rejected parameters %q: %v", parameters, err) + } + } +} + +func TestHasPath(t *testing.T) { + parameters := map[string]any{ + "goal": map[string]any{"target": nil, "sla": map[string]any{"ttft_ms": 1.0}}, + "workload": map[string]any{"isl": 8.0}, + } + for path, want := range map[string]bool{ + "goal.target": true, // a null still names the key + "goal.sla": true, + "goal.sla.ttft_ms": true, + "goal.sla.itl_ms": false, + "workload.isl": true, + "workload.osl": false, + "sweep.max_trials": false, + "goal.target.deeper": false, + "workload.isl.deeper": false, + } { + if got := has(parameters, strings.Split(path, ".")...); got != want { + t.Errorf("has(%s) = %v, want %v", path, got, want) + } + } + if has(nil, "goal") { + t.Error("nothing is named in nil parameters") + } +} + +func TestNativeMetricNames(t *testing.T) { + for metric, want := range map[v1beta2.ObjectiveMetric]string{ + "Throughput": "throughput", + "ThroughputPerGPU": "throughput_per_gpu", + "ThroughputPerUser": "throughput_per_user", + "TTFT": "ttft", + "E2ELatency": "e2e_latency", + "Goodput": "goodput", + "GoodputPerGPU": "goodput_per_gpu", + } { + if got := nativeMetric(metric); got != want { + t.Errorf("nativeMetric(%s) = %q, want %q", metric, got, want) + } + } +} diff --git a/deploy/operator/internal/controller/aisimulate/paths.go b/deploy/operator/internal/controller/aisimulate/paths.go new file mode 100644 index 000000000000..8a2c13264f2a --- /dev/null +++ b/deploy/operator/internal/controller/aisimulate/paths.go @@ -0,0 +1,75 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package aisimulate + +import "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + +// Native input paths, in the snake_case spelling of the upstream AISimulate contract. They +// address the same native key when the underlay is built and when derived defaults are read, +// and are passed to the unstructured helpers, which must not modify them. +var ( + searchSpaceModelName = []string{"search_space", "model_name"} + searchSpaceBackend = []string{"search_space", "backend"} + searchSpaceHardwareSKU = []string{"search_space", "hardware_sku"} + searchSpaceGPUBudget = []string{"search_space", "gpu_budget"} + + workloadPath = []string{"workload"} + workloadISL = []string{"workload", "isl"} + workloadOSL = []string{"workload", "osl"} + workloadNumRequestRatio = []string{"workload", "num_request_ratio"} + workloadRequestRate = []string{"workload", "request_rate"} + workloadConcurrency = []string{"workload", "concurrency"} + workloadKVLoadRatio = []string{"workload", "kv_load_ratio"} + workloadRequestCount = []string{"workload", "request_count"} + + goalTarget = []string{"goal", "target"} + goalParetoObjectives = []string{"goal", "pareto_objectives"} + goalSLATTFTMs = []string{"goal", "sla", "ttft_ms"} + goalSLAITLMs = []string{"goal", "sla", "itl_ms"} + goalSLAE2EMs = []string{"goal", "sla", "e2e_ms"} + goalStrictSLA = []string{"goal", "strict_sla"} + + sweepMaxTrials = []string{"sweep", "max_trials"} +) + +// TargetPareto is the goal.target of a multi-objective search. +const TargetPareto = "pareto" + +// has reports whether obj contains the path, even with a null value. Applied to +// spec.aisimulate.parameters it tells whether the caller named a key, including to delete +// it. +func has(obj map[string]any, path ...string) bool { + _, found, _ := unstructured.NestedFieldNoCopy(obj, path...) + return found +} + +// HardwareNamed reports whether the native input names both a GPU SKU and a GPU budget, as the +// search needs. A null or blank value does not count: the key is deleted or left empty. +func HardwareNamed(input map[string]any) bool { + return hasValue(input, searchSpaceHardwareSKU...) && hasValue(input, searchSpaceGPUBudget...) +} + +// hasValue reports whether obj holds a non-null, non-blank value at the path. +func hasValue(obj map[string]any, path ...string) bool { + value, found, err := unstructured.NestedFieldNoCopy(obj, path...) + if err != nil || !found || value == nil { + return false + } + text, isString := value.(string) + return !isString || text != "" +} diff --git a/deploy/operator/internal/controller/dynamographdeploymentrequest_controller.go b/deploy/operator/internal/controller/dynamographdeploymentrequest_controller.go index 67f35c3a1bc7..1d24fd6060a8 100644 --- a/deploy/operator/internal/controller/dynamographdeploymentrequest_controller.go +++ b/deploy/operator/internal/controller/dynamographdeploymentrequest_controller.go @@ -26,7 +26,6 @@ import ( "io" "reflect" "sort" - "strings" "text/template" "time" @@ -1503,52 +1502,11 @@ func (r *DynamoGraphDeploymentRequestReconciler) validateGPUHardwareInfo(ctx con "or set spec.hardware.{gpuSku,vramMb,numGpusPerNode} explicitly.") } -// GetGPUDiscoveryFailureReason classifies a GPU discovery error and -// returns a stable, actionable reason string suitable for structured logging. -// -// The classification is based on known error message patterns produced during: -// - DCGM exporter pod discovery -// - Helm-based GPU operator and DCGM discovery -// - Metrics scraping -// - Prometheus parsing -// -// If the error does not match any known category, "unknown" is returned. +// GetGPUDiscoveryFailureReason classifies a GPU discovery error into a stable reason string +// for structured logging. The classification lives in the gpu package so the v1beta2 +// admission path reports the same reasons. func GetGPUDiscoveryFailureReason(err error) string { - const unknownReason = "unknown" - - if err == nil { - return unknownReason - } - errMsg := strings.ToLower(err.Error()) - - switch { - case strings.Contains(errMsg, "list pods"): - return "failed to list DCGM exporter pods (RBAC/cluster connectivity issue)" - case strings.Contains(errMsg, "gpu operator is not installed"): - return "GPU Operator not installed in expected namespace" - case strings.Contains(errMsg, "helm init failed"): - return "failed to initialize Helm client (RBAC, kubeconfig, or Helm driver issue)" - case strings.Contains(errMsg, "timeout waiting for dcgm exporter pods"): - return "timeout while waiting for DCGM exporter pods to become ready" - case strings.Contains(errMsg, "http get"): - return "failed to reach DCGM metrics endpoint on pod (network/port issue)" - case strings.Contains(errMsg, "metrics endpoint") && - strings.Contains(errMsg, "status"): - return "DCGM pod metrics endpoint returned non-200 status" - case strings.Contains(errMsg, "parse prometheus metrics"): - return "failed to parse dcgm Prometheus metrics (invalid format)" - case strings.Contains(errMsg, "no gpus detected"): - return "no GPUs detected in dcgm metrics (GPU model or metrics missing)" - case strings.Contains(errMsg, "dcgm is not enabled in the GPU Operator"): - return "DCGM is not enabled in the GPU Operator (check GPU Operator configuration and permissions)" - case strings.Contains(errMsg, "failed to scrape any dcgm exporter pod"): - return "failed to scrape any dcgm exporter pod (check DCGM exporter pod status and network connectivity)" - case strings.Contains(errMsg, "no gpu metrics could be parsed from any dcgm pod"): - return "no GPU metrics could be parsed from any DCGM pod (check DCGM exporter pod status and network connectivity)" - case strings.Contains(errMsg, "failed to create helm path"): - return "failed to initialize Helm client (RBAC, kubeconfig, or Helm driver issue)" - } - return unknownReason + return gpu.DiscoveryFailureReason(err) } // createProfilingJob creates a Kubernetes Job for profiling using SyncResource. @@ -1999,41 +1957,21 @@ func (r *DynamoGraphDeploymentRequestReconciler) discoverHardwareForEnrichment(c return nil, nil } - var dcgmErr error - if r.GPUDiscovery != nil { - discoveredInfo, err := r.GPUDiscovery.DiscoverGPUsFromDCGMFiltered(ctx, reader, r.GPUDiscoveryCache, hw.GPUSKU) - if err == nil { - return discoveredInfo, nil - } - dcgmErr = err - - reason := GetGPUDiscoveryFailureReason(err) - logger.Info("DCGM discovery failed, falling back to node-label discovery", - "reason", reason, "error", err.Error()) + discoverer := &gpu.Discoverer{ + Reader: reader, + DCGM: r.GPUDiscovery, + Cache: r.GPUDiscoveryCache, + NodeLabels: r.gpuDiscoveryEnabled, } - - if !r.gpuDiscoveryEnabled() { + info, err := discoverer.Discover(ctx, hw.GPUSKU) + if err != nil { if discoveryRequired { - if dcgmErr != nil { - return nil, fmt.Errorf("auto-discovery failed: %w", dcgmErr) - } - return nil, fmt.Errorf("auto-discovery failed: node-label discovery is disabled") + return nil, err } - logger.Info("Optional hardware metadata discovery skipped because node-label discovery is disabled") + logger.Info("Optional hardware metadata discovery unavailable; leaving unset fields unchanged", "error", err.Error()) return nil, nil } - - discoveredInfo, err := gpu.DiscoverGPUsFiltered(ctx, reader, hw.GPUSKU) - if err == nil { - return discoveredInfo, nil - } - - logger.Info("Node-label discovery also failed", "error", err.Error()) - if discoveryRequired { - return nil, fmt.Errorf("auto-discovery failed: %w", err) - } - logger.Info("Optional hardware metadata discovery unavailable; leaving unset fields unchanged") - return nil, nil + return info, nil } // extractModelCachePVCConfig reads model cache PVC settings from the typed v1beta1 spec. diff --git a/deploy/operator/internal/controller/dynamographdeploymentrequest_v1beta2_controller.go b/deploy/operator/internal/controller/dynamographdeploymentrequest_v1beta2_controller.go new file mode 100644 index 000000000000..c88d2f1ec4af --- /dev/null +++ b/deploy/operator/internal/controller/dynamographdeploymentrequest_v1beta2_controller.go @@ -0,0 +1,421 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package controller + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "sort" + "strconv" + "strings" + "time" + + configv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/config/v1alpha1" + nvidiacomv1beta2 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta2" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/controller/aisimulate" + commonController "github.com/ai-dynamo/dynamo/deploy/operator/internal/controller_common" + corev1 "k8s.io/api/core/v1" + apiequality "k8s.io/apimachinery/pkg/api/equality" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/log" +) + +const ( + // dgdrV1beta2GenerationAnnotation records the Request generation a Run was created for. It is + // diagnostic only: the generation the controller acts on is the one in the immutable Run name. + dgdrV1beta2GenerationAnnotation = "nvidia.com/dgdr-request-generation" + + // dgdrRunControllerUIDIndex indexes Runs by the UID of their controller OwnerReference. The + // OwnerReference is the only definition of which Runs belong to a Request. + dgdrRunControllerUIDIndex = "dynamo.nvidia.com/dgdr-run-controller-uid" + + // dgdrV1beta2RetainedRuns is how many terminal Runs a Request keeps. Older ones are deleted. + dgdrV1beta2RetainedRuns = 5 + + // DGDRAcceptedConditionV1beta2 reports whether the current Request generation has been + // materialized as an immutable Run. Execution state belongs to the Run, not to this condition. + DGDRAcceptedConditionV1beta2 = "Accepted" + + dgdrReasonRunCreated = "RunCreated" + dgdrReasonInvalidSpec = "InvalidSpec" + dgdrReasonWaitingForActiveRun = "WaitingForActiveRun" + dgdrReasonRunNameConflict = "RunNameConflict" + + // dgdrV1beta2MaxRunNameLength is the longest object name Kubernetes accepts. + dgdrV1beta2MaxRunNameLength = 253 + + // dgdrV1beta2ConflictRetry is how long to wait before looking again at a Run name held by another owner. + dgdrV1beta2ConflictRetry = 5 * time.Second +) + +// DynamoGraphDeploymentRequestV1beta2Reconciler turns an accepted native v1beta2 Request into an +// immutable Run. It never reads the cluster for hardware: admission has already supplied it, and a +// Request that still lacks it is reported instead of searched for. The Run controller executes the Run. +type DynamoGraphDeploymentRequestV1beta2Reconciler struct { + client.Client + Config *configv1alpha1.OperatorConfiguration + RuntimeConfig *commonController.RuntimeConfig +} + +// +kubebuilder:rbac:groups=nvidia.com,resources=dynamographdeploymentrequests,verbs=get;list;watch +// +kubebuilder:rbac:groups=nvidia.com,resources=dynamographdeploymentrequests/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=nvidia.com,resources=dynamographdeploymentruns,verbs=get;list;watch;create;delete + +// Reconcile is level based: once no older Run is active it creates the Run of the latest generation +// only, so generations edited in between are coalesced and never get a Run of their own. It records +// the active and retained Runs in the Request status before it deletes any retired Run. +func (r *DynamoGraphDeploymentRequestV1beta2Reconciler) Reconcile( + ctx context.Context, + req ctrl.Request, +) (ctrl.Result, error) { + logger := log.FromContext(ctx) + + request := &nvidiacomv1beta2.DynamoGraphDeploymentRequest{} + if err := r.Get(ctx, req.NamespacedName, request); err != nil { + return ctrl.Result{}, client.IgnoreNotFound(err) + } + // A native v1beta1 Request is a read-only carrier here; the v1beta1 reconciler owns it. + if request.DeletionTimestamp != nil || request.Spec.V1Beta1 != nil { + return ctrl.Result{}, nil + } + original := request.DeepCopy() + result := ctrl.Result{} + + runs, err := r.ownedRuns(ctx, request) + if err != nil { + return ctrl.Result{}, err + } + + wantName := dgdrV1beta2RunName(request) + var current *nvidiacomv1beta2.DynamoGraphDeploymentRun + for i := range runs { + if runs[i].Name == wantName { + current = &runs[i] + } + } + + active, _, err := dgdrV1beta2SplitRuns(request.Name, runs) + if err != nil { + return ctrl.Result{}, err + } + + accepted := current != nil + condition := dgdrV1beta2Condition(request, metav1.ConditionTrue, dgdrReasonRunCreated, + fmt.Sprintf("Run %q exists for generation %d", wantName, request.Generation)) + if current == nil { + if active != nil { + condition = dgdrV1beta2Condition(request, metav1.ConditionFalse, dgdrReasonWaitingForActiveRun, + fmt.Sprintf("Run %q is still active; the Run for generation %d starts when it finishes", + active.Name, request.Generation)) + } else if run, buildErr := r.buildRun(request); buildErr != nil { + logger.Info("Request cannot be turned into a Run", "reason", buildErr.Error()) + condition = dgdrV1beta2Condition(request, metav1.ConditionFalse, dgdrReasonInvalidSpec, buildErr.Error()) + } else { + err := r.Create(ctx, run) + switch { + case err == nil: + runs = append(runs, *run) + accepted = true + case apierrors.IsAlreadyExists(err): + // A stale cache, or a Run left by an earlier Request of the same name that is still being + // garbage collected. Only a Run this Request controls counts as ours. + existing := &nvidiacomv1beta2.DynamoGraphDeploymentRun{} + if getErr := r.Get(ctx, client.ObjectKeyFromObject(run), existing); getErr != nil { + return ctrl.Result{}, fmt.Errorf("get existing Run %q: %w", wantName, getErr) + } + if metav1.IsControlledBy(existing, request) { + runs = append(runs, *existing) + accepted = true + } else { + condition = dgdrV1beta2Condition(request, metav1.ConditionFalse, dgdrReasonRunNameConflict, + fmt.Sprintf("Run %q exists and is owned by another object; trying again once it is gone", wantName)) + result.RequeueAfter = dgdrV1beta2ConflictRetry + } + default: + return ctrl.Result{}, fmt.Errorf("create Run %q: %w", wantName, err) + } + } + } + + active, terminal, err := dgdrV1beta2SplitRuns(request.Name, runs) + if err != nil { + return ctrl.Result{}, err + } + keep := min(len(terminal), dgdrV1beta2RetainedRuns) + retired := terminal[keep:] + + if accepted { + request.Status.ObservedGeneration = request.Generation + } + request.Status.ActiveRunRef = nil + if active != nil { + request.Status.ActiveRunRef = &corev1.LocalObjectReference{Name: active.Name} + } + request.Status.RecentRunRefs = nil + for i := 0; i < keep; i++ { + request.Status.RecentRunRefs = append(request.Status.RecentRunRefs, corev1.LocalObjectReference{Name: terminal[i].Name}) + } + meta.SetStatusCondition(&request.Status.Conditions, condition) + + if !apiequality.Semantic.DeepEqual(original.Status, request.Status) { + if err := r.Status().Patch(ctx, request, client.MergeFrom(original)); err != nil { + return ctrl.Result{}, fmt.Errorf("update Request status: %w", err) + } + } + + // The status no longer references these Runs, so they may go. If a deletion fails, an unreferenced + // child stays for the next pass; the status never points at a Run that was already deleted. + for i := range retired { + if err := r.Delete(ctx, &retired[i]); err != nil && !apierrors.IsNotFound(err) { + return ctrl.Result{}, fmt.Errorf("delete retired Run %q: %w", retired[i].Name, err) + } + } + return result, nil +} + +// buildRun resolves the optional inputs of the Request into an immutable Run spec. The effective +// AISimulate input is frozen into aisimulate.parameters: that object is the complete resolved input. +// Run execution decodes it directly and never lowers the Run again, because the +// merge-patch deletions (null) that shaped it are not part of the stored result. +func (r *DynamoGraphDeploymentRequestV1beta2Reconciler) buildRun( + request *nvidiacomv1beta2.DynamoGraphDeploymentRequest, +) (*nvidiacomv1beta2.DynamoGraphDeploymentRun, error) { + spec := &request.Spec + if spec.AISimulate == nil || spec.AISimulate.Image == "" { + return nil, errors.New("spec.aisimulate.image is required") + } + version, err := aisimulate.ResolveVersion(spec.AISimulate.Image, spec.AISimulate.VersionOverride) + if err != nil { + return nil, err + } + input, err := aisimulate.Lower(spec) + if err != nil { + return nil, err + } + if !aisimulate.HardwareNamed(input) { + return nil, errors.New("the effective AISimulate input names no GPU SKU or budget; set spec.hardware.gpu") + } + raw, err := json.Marshal(input) + if err != nil { + return nil, fmt.Errorf("encode the effective AISimulate input: %w", err) + } + + frozen := spec.NativeSearchInput.DeepCopy() + frozen.AISimulate = &nvidiacomv1beta2.AISimulateSpec{ + Image: spec.AISimulate.Image, + VersionOverride: version, + Parameters: &runtime.RawExtension{Raw: raw}, + } + run := &nvidiacomv1beta2.DynamoGraphDeploymentRun{ + ObjectMeta: metav1.ObjectMeta{ + Name: dgdrV1beta2RunName(request), + Namespace: request.Namespace, + Annotations: map[string]string{dgdrV1beta2GenerationAnnotation: strconv.FormatInt(request.Generation, 10)}, + }, + Spec: nvidiacomv1beta2.DynamoGraphDeploymentRunSpec{NativeSearchInput: *frozen}, + } + if err := controllerutil.SetControllerReference(request, run, r.Scheme()); err != nil { + return nil, fmt.Errorf("set Run owner: %w", err) + } + return run, nil +} + +// ownedRuns returns the Runs this Request controls. The controller OwnerReference is the only +// source of ownership: the cache index selects candidates and IsControlledBy is the exact check. +func (r *DynamoGraphDeploymentRequestV1beta2Reconciler) ownedRuns( + ctx context.Context, + request *nvidiacomv1beta2.DynamoGraphDeploymentRequest, +) ([]nvidiacomv1beta2.DynamoGraphDeploymentRun, error) { + list := &nvidiacomv1beta2.DynamoGraphDeploymentRunList{} + if err := r.List(ctx, list, client.InNamespace(request.Namespace), + client.MatchingFields{dgdrRunControllerUIDIndex: string(request.UID)}); err != nil { + return nil, fmt.Errorf("list Runs: %w", err) + } + owned := make([]nvidiacomv1beta2.DynamoGraphDeploymentRun, 0, len(list.Items)) + for i := range list.Items { + if metav1.IsControlledBy(&list.Items[i], request) { + owned = append(owned, list.Items[i]) + } + } + return owned, nil +} + +// SetupWithManager registers the controller. It watches the Runs it owns, so a finished Run wakes +// the Request that was waiting for it. +func (r *DynamoGraphDeploymentRequestV1beta2Reconciler) SetupWithManager(mgr ctrl.Manager) error { + if err := mgr.GetFieldIndexer().IndexField( + context.Background(), + &nvidiacomv1beta2.DynamoGraphDeploymentRun{}, + dgdrRunControllerUIDIndex, + dgdrRunControllerUIDs, + ); err != nil { + return fmt.Errorf("index Runs by controller UID: %w", err) + } + return ctrl.NewControllerManagedBy(mgr). + Named("dynamographdeploymentrequest-v1beta2"). + For(&nvidiacomv1beta2.DynamoGraphDeploymentRequest{}). + Owns(&nvidiacomv1beta2.DynamoGraphDeploymentRun{}). + WithEventFilter(commonController.EphemeralDeploymentEventFilter(r.Config, r.RuntimeConfig)). + Complete(r) +} + +// dgdrRunControllerUIDs is the index function behind dgdrRunControllerUIDIndex. +func dgdrRunControllerUIDs(obj client.Object) []string { + owner := metav1.GetControllerOf(obj) + if owner == nil { + return nil + } + return []string{string(owner.UID)} +} + +// dgdrV1beta2RunName names the Run of one Request generation. A metadata-only update keeps the +// generation, so it never creates a second Run. +func dgdrV1beta2RunName(request *nvidiacomv1beta2.DynamoGraphDeploymentRequest) string { + return dgdrV1beta2RunNameFor(request.Name, request.Generation) +} + +// dgdrV1beta2RunNameFor is - whenever that fits in an object name. A longer +// name keeps the generation suffix and a stable hash of the full Request name, and truncates only the +// Request-name part, so every valid Request name has a valid Run name and two long names that share a +// prefix still get different Run names. +func dgdrV1beta2RunNameFor(requestName string, generation int64) string { + suffix := "-" + strconv.FormatInt(generation, 10) + if len(requestName)+len(suffix) <= dgdrV1beta2MaxRunNameLength { + return requestName + suffix + } + sum := sha256.Sum256([]byte(requestName)) + hash := hex.EncodeToString(sum[:6]) + maxPrefix := dgdrV1beta2MaxRunNameLength - len(suffix) - 1 - len(hash) + prefix := strings.TrimRight(requestName[:maxPrefix], "-.") + return prefix + "-" + hash + suffix +} + +func dgdrV1beta2Condition( + request *nvidiacomv1beta2.DynamoGraphDeploymentRequest, + status metav1.ConditionStatus, + reason, message string, +) metav1.Condition { + return metav1.Condition{ + Type: DGDRAcceptedConditionV1beta2, + Status: status, + Reason: reason, + Message: message, + ObservedGeneration: request.Generation, + } +} + +// dgdrV1beta2IsTerminal reports whether the Run has published its Completed condition. The conditions +// are authoritative; the phase only describes how a completed Run ended. +func dgdrV1beta2IsTerminal(run *nvidiacomv1beta2.DynamoGraphDeploymentRun) bool { + return meta.IsStatusConditionTrue(run.Status.Conditions, nvidiacomv1beta2.DGDRRunConditionCompleted) +} + +// dgdrV1beta2RunGeneration reads the Request generation from the immutable Run name. The name is +// accepted only if it is exactly the name this Request derives for that generation; anything else is an +// error, never generation zero. +func dgdrV1beta2RunGeneration(requestName string, run *nvidiacomv1beta2.DynamoGraphDeploymentRun) (int64, error) { + index := strings.LastIndex(run.Name, "-") + if index < 0 { + return 0, fmt.Errorf("run %q has no Request generation suffix", run.Name) + } + generation, err := strconv.ParseInt(run.Name[index+1:], 10, 64) + if err != nil { + return 0, fmt.Errorf("run %q has no valid Request generation: %w", run.Name, err) + } + if expected := dgdrV1beta2RunNameFor(requestName, generation); run.Name != expected { + return 0, fmt.Errorf("run %q is not the Run name of generation %d of Request %q (expected %q)", + run.Name, generation, requestName, expected) + } + return generation, nil +} + +type dgdrV1beta2RunEntry struct { + run nvidiacomv1beta2.DynamoGraphDeploymentRun + generation int64 +} + +// dgdrV1beta2Newer orders Runs newest first: by Request generation, then creation time, then name. +func dgdrV1beta2Newer(a, b *dgdrV1beta2RunEntry) bool { + if a.generation != b.generation { + return a.generation > b.generation + } + if !a.run.CreationTimestamp.Equal(&b.run.CreationTimestamp) { + return b.run.CreationTimestamp.Before(&a.run.CreationTimestamp) + } + return a.run.Name > b.run.Name +} + +// dgdrV1beta2SplitRuns returns the newest non-terminal Run, if any, and the terminal Runs newest first. +func dgdrV1beta2SplitRuns( + requestName string, + runs []nvidiacomv1beta2.DynamoGraphDeploymentRun, +) (*nvidiacomv1beta2.DynamoGraphDeploymentRun, []nvidiacomv1beta2.DynamoGraphDeploymentRun, error) { + entries := make([]dgdrV1beta2RunEntry, 0, len(runs)) + for i := range runs { + generation, err := dgdrV1beta2RunGeneration(requestName, &runs[i]) + if err != nil { + return nil, nil, err + } + entries = append(entries, dgdrV1beta2RunEntry{run: runs[i], generation: generation}) + } + sort.SliceStable(entries, func(i, j int) bool { return dgdrV1beta2Newer(&entries[i], &entries[j]) }) + var active *nvidiacomv1beta2.DynamoGraphDeploymentRun + var terminal []nvidiacomv1beta2.DynamoGraphDeploymentRun + for i := range entries { + switch { + case !dgdrV1beta2IsTerminal(&entries[i].run): + if active == nil { + active = &entries[i].run + } + default: + terminal = append(terminal, entries[i].run) + } + } + return active, terminal, nil +} + +// requireDGDRV1beta2API fails when the cluster does not serve the v1beta2 Request, Run and +// Candidate APIs. Without them either the manager caches cannot sync or a Run's publisher +// cannot start. +func requireDGDRV1beta2API(mgr ctrl.Manager) error { + group, version := nvidiacomv1beta2.GroupVersion.Group, nvidiacomv1beta2.GroupVersion.Version + for _, kind := range []string{ + "DynamoGraphDeploymentRequest", "DynamoGraphDeploymentRun", "DynamoGraphDeploymentCandidate", + } { + _, err := mgr.GetRESTMapper().RESTMapping(schema.GroupKind{Group: group, Kind: kind}, version) + if meta.IsNoMatchError(err) { + return fmt.Errorf("--dgdr-v1beta2-enabled is set but the cluster does not serve %s/%s %s; "+ + "install the v1beta2 CRDs (dgdrV1Beta2.enabled)", group, version, kind) + } + if err != nil { + return fmt.Errorf("check that %s/%s %s is served: %w", group, version, kind, err) + } + } + return nil +} diff --git a/deploy/operator/internal/controller/dynamographdeploymentrequest_v1beta2_controller_envtest_test.go b/deploy/operator/internal/controller/dynamographdeploymentrequest_v1beta2_controller_envtest_test.go new file mode 100644 index 000000000000..2de8042f8ce2 --- /dev/null +++ b/deploy/operator/internal/controller/dynamographdeploymentrequest_v1beta2_controller_envtest_test.go @@ -0,0 +1,154 @@ +//go:build !clustertest + +/* + * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package controller + +import ( + "context" + "testing" + "time" + + nvidiacomv1beta2 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta2" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/controller/aisimulate" + . "github.com/onsi/gomega" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// This test runs the controller under a real manager, so only the watch events (a Request edit, a Run +// status update, a deleted Run) can make it reconcile. It fails if SetupWithManager loses the Owns +// watch, an event predicate drops Run status updates, or the owner mapping breaks. It also covers the +// API server and admission side: the Request is created model-only and admission supplies the image +// and the hardware. +func TestDGDRV1beta2ControllerReactsToRequestAndRunEvents(t *testing.T) { + g := NewWithT(t) + env := sharedEnv.ForTest(t) + c := env.Client() + ctx := context.Background() + + env.StartManager(func(mgr ctrl.Manager) error { + return (&DynamoGraphDeploymentRequestV1beta2Reconciler{ + Client: mgr.GetClient(), + Config: env.OperatorConfig(), + RuntimeConfig: env.RuntimeConfig(), + }).SetupWithManager(mgr) + }) + + const timeout = 30 * time.Second + const poll = 200 * time.Millisecond + + request := &nvidiacomv1beta2.DynamoGraphDeploymentRequest{ + ObjectMeta: metav1.ObjectMeta{Name: "watched", Namespace: env.Namespace()}, + Spec: nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec{ + NativeSearchInput: nvidiacomv1beta2.NativeSearchInput{ + Model: &nvidiacomv1beta2.ModelSpec{Name: "Qwen/Qwen3-32B"}, + }, + }, + } + key := client.ObjectKeyFromObject(request) + getRequest := func(g Gomega) *nvidiacomv1beta2.DynamoGraphDeploymentRequest { + live := &nvidiacomv1beta2.DynamoGraphDeploymentRequest{} + g.Expect(c.Get(ctx, key, live)).To(Succeed()) + return live + } + accepted := func(live *nvidiacomv1beta2.DynamoGraphDeploymentRequest) *metav1.Condition { + return meta.FindStatusCondition(live.Status.Conditions, DGDRAcceptedConditionV1beta2) + } + getRun := func(g Gomega, name string) *nvidiacomv1beta2.DynamoGraphDeploymentRun { + run := &nvidiacomv1beta2.DynamoGraphDeploymentRun{} + g.Expect(c.Get(ctx, types.NamespacedName{Namespace: env.Namespace(), Name: name}, run)).To(Succeed()) + return run + } + + t.Log("Creating the Request produces its first Run without a direct Reconcile call") + g.Expect(c.Create(ctx, request)).To(Succeed()) + var firstName string + g.Eventually(func(g Gomega) { + live := getRequest(g) + g.Expect(live.Status.ActiveRunRef).NotTo(BeNil()) + g.Expect(accepted(live)).NotTo(BeNil()) + g.Expect(accepted(live).Status).To(Equal(metav1.ConditionTrue)) + g.Expect(live.Status.ObservedGeneration).To(Equal(live.Generation)) + firstName = live.Status.ActiveRunRef.Name + }).WithTimeout(timeout).WithPolling(poll).Should(Succeed()) + + first := getRun(g, firstName) + g.Expect(metav1.IsControlledBy(first, getRequest(g))).To(BeTrue()) + g.Expect(first.Spec.Hardware).NotTo(BeNil()) + g.Expect(first.Spec.AISimulate.Image).To(Equal(controllerAISimulateImage)) + g.Expect(first.Spec.AISimulate.VersionOverride).To(Equal("1.2.0")) + input, err := aisimulate.DecodeParameters(first.Spec.AISimulate.Parameters) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(input).To(HaveKey("search_space")) + + t.Log("Editing the Request while its Run is active is reported and starts nothing") + g.Eventually(func(g Gomega) { + live := getRequest(g) + live.Spec.Model.Name = "Qwen/Qwen3-30B-A3B" + g.Expect(c.Update(ctx, live)).To(Succeed()) + }).WithTimeout(timeout).WithPolling(poll).Should(Succeed()) + g.Eventually(func(g Gomega) { + live := getRequest(g) + condition := accepted(live) + g.Expect(condition).NotTo(BeNil()) + g.Expect(condition.Status).To(Equal(metav1.ConditionFalse)) + g.Expect(condition.Reason).To(Equal(dgdrReasonWaitingForActiveRun)) + g.Expect(condition.ObservedGeneration).To(Equal(live.Generation)) + }).WithTimeout(timeout).WithPolling(poll).Should(Succeed()) + + t.Log("Only the Run status update wakes the Request: Completed releases the next generation") + g.Eventually(func(g Gomega) { + run := getRun(g, firstName) + run.Status.Phase = nvidiacomv1beta2.DGDRRunPhase("Succeeded") + meta.SetStatusCondition(&run.Status.Conditions, metav1.Condition{ + Type: nvidiacomv1beta2.DGDRRunConditionCompleted, + Status: metav1.ConditionTrue, + Reason: "Finished", + }) + g.Expect(c.Status().Update(ctx, run)).To(Succeed()) + }).WithTimeout(timeout).WithPolling(poll).Should(Succeed()) + + var secondName string + var secondUID types.UID + g.Eventually(func(g Gomega) { + live := getRequest(g) + g.Expect(live.Status.ActiveRunRef).NotTo(BeNil()) + g.Expect(live.Status.ActiveRunRef.Name).NotTo(Equal(firstName)) + g.Expect(live.Status.ObservedGeneration).To(Equal(live.Generation)) + g.Expect(accepted(live).Status).To(Equal(metav1.ConditionTrue)) + g.Expect(live.Status.RecentRunRefs).To(HaveLen(1)) + g.Expect(live.Status.RecentRunRefs[0].Name).To(Equal(firstName)) + second := getRun(g, live.Status.ActiveRunRef.Name) + g.Expect(second.Spec.Model.Name).To(Equal("Qwen/Qwen3-30B-A3B")) + secondName, secondUID = second.Name, second.UID + }).WithTimeout(timeout).WithPolling(poll).Should(Succeed()) + + t.Log("Deleting the active Run wakes the Request through the owner mapping and it is recreated") + g.Expect(c.Delete(ctx, getRun(g, secondName))).To(Succeed()) + g.Eventually(func(g Gomega) { + recreated := getRun(g, secondName) + g.Expect(recreated.UID).NotTo(Equal(secondUID)) + live := getRequest(g) + g.Expect(live.Status.ActiveRunRef).NotTo(BeNil()) + g.Expect(live.Status.ActiveRunRef.Name).To(Equal(secondName)) + }).WithTimeout(timeout).WithPolling(poll).Should(Succeed()) +} diff --git a/deploy/operator/internal/controller/dynamographdeploymentrequest_v1beta2_controller_test.go b/deploy/operator/internal/controller/dynamographdeploymentrequest_v1beta2_controller_test.go new file mode 100644 index 000000000000..71735c93ef92 --- /dev/null +++ b/deploy/operator/internal/controller/dynamographdeploymentrequest_v1beta2_controller_test.go @@ -0,0 +1,621 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package controller + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "strings" + "testing" + + nvidiacomv1beta2 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta2" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/controller/aisimulate" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" +) + +type dgdrV1beta2Fixture struct { + t *testing.T + client client.Client + reconciler *DynamoGraphDeploymentRequestV1beta2Reconciler +} + +func newDGDRV1beta2Fixture(t *testing.T, objects ...client.Object) *dgdrV1beta2Fixture { + t.Helper() + return newDGDRV1beta2FixtureWithInterceptor(t, interceptor.Funcs{}, objects...) +} + +func newDGDRV1beta2FixtureWithInterceptor( + t *testing.T, + funcs interceptor.Funcs, + objects ...client.Object, +) *dgdrV1beta2Fixture { + t.Helper() + scheme := runtime.NewScheme() + if err := nvidiacomv1beta2.AddToScheme(scheme); err != nil { + t.Fatalf("add v1beta2 to scheme: %v", err) + } + c := fake.NewClientBuilder(). + WithScheme(scheme). + WithObjects(objects...). + WithIndex(&nvidiacomv1beta2.DynamoGraphDeploymentRun{}, dgdrRunControllerUIDIndex, dgdrRunControllerUIDs). + WithStatusSubresource(&nvidiacomv1beta2.DynamoGraphDeploymentRequest{}, &nvidiacomv1beta2.DynamoGraphDeploymentRun{}). + WithInterceptorFuncs(funcs). + Build() + return &dgdrV1beta2Fixture{t: t, client: c, reconciler: &DynamoGraphDeploymentRequestV1beta2Reconciler{Client: c}} +} + +func (f *dgdrV1beta2Fixture) reconcile() { + f.t.Helper() + if err := f.tryReconcile(); err != nil { + f.t.Fatalf("reconcile: %v", err) + } +} + +func (f *dgdrV1beta2Fixture) tryReconcile() error { + f.t.Helper() + _, err := f.reconciler.Reconcile(f.t.Context(), ctrl.Request{ + NamespacedName: types.NamespacedName{Namespace: "default", Name: "search"}, + }) + return err +} + +func (f *dgdrV1beta2Fixture) request() *nvidiacomv1beta2.DynamoGraphDeploymentRequest { + f.t.Helper() + request := &nvidiacomv1beta2.DynamoGraphDeploymentRequest{} + if err := f.client.Get(f.t.Context(), types.NamespacedName{Namespace: "default", Name: "search"}, request); err != nil { + f.t.Fatalf("get request: %v", err) + } + return request +} + +func (f *dgdrV1beta2Fixture) runNames() []string { + f.t.Helper() + list := &nvidiacomv1beta2.DynamoGraphDeploymentRunList{} + if err := f.client.List(f.t.Context(), list, client.InNamespace("default")); err != nil { + f.t.Fatalf("list runs: %v", err) + } + names := make([]string, 0, len(list.Items)) + for _, run := range list.Items { + names = append(names, run.Name) + } + return names +} + +func (f *dgdrV1beta2Fixture) updateRunStatus(name string, mutate func(*nvidiacomv1beta2.DynamoGraphDeploymentRunStatus)) { + f.t.Helper() + run := &nvidiacomv1beta2.DynamoGraphDeploymentRun{} + if err := f.client.Get(f.t.Context(), types.NamespacedName{Namespace: "default", Name: name}, run); err != nil { + f.t.Fatalf("get run %s: %v", name, err) + } + mutate(&run.Status) + if err := f.client.Status().Update(f.t.Context(), run); err != nil { + f.t.Fatalf("update status of run %s: %v", name, err) + } +} + +// finish ends a Run the way the Run controller does: it publishes Completed and the outcome phase. +func (f *dgdrV1beta2Fixture) finish(name, phase string) { + f.t.Helper() + f.updateRunStatus(name, func(status *nvidiacomv1beta2.DynamoGraphDeploymentRunStatus) { + status.Phase = nvidiacomv1beta2.DGDRRunPhase(phase) + setCompletedForTest(status) + }) +} + +// setPhaseOnly writes a terminal phase without the Completed condition. +func (f *dgdrV1beta2Fixture) setPhaseOnly(name, phase string) { + f.t.Helper() + f.updateRunStatus(name, func(status *nvidiacomv1beta2.DynamoGraphDeploymentRunStatus) { + status.Phase = nvidiacomv1beta2.DGDRRunPhase(phase) + }) +} + +// setCompletedOnly publishes Completed without a terminal phase. +func (f *dgdrV1beta2Fixture) setCompletedOnly(name string) { + f.t.Helper() + f.updateRunStatus(name, setCompletedForTest) +} + +func setCompletedForTest(status *nvidiacomv1beta2.DynamoGraphDeploymentRunStatus) { + meta.SetStatusCondition(&status.Conditions, metav1.Condition{ + Type: nvidiacomv1beta2.DGDRRunConditionCompleted, + Status: metav1.ConditionTrue, + Reason: "Finished", + }) +} + +func (f *dgdrV1beta2Fixture) bumpGeneration(generation int64) { + f.t.Helper() + request := f.request() + request.Generation = generation + if err := f.client.Update(f.t.Context(), request); err != nil { + f.t.Fatalf("bump generation: %v", err) + } +} + +func dgdrV1beta2Request(withHardware bool) *nvidiacomv1beta2.DynamoGraphDeploymentRequest { + request := &nvidiacomv1beta2.DynamoGraphDeploymentRequest{ + ObjectMeta: metav1.ObjectMeta{Name: "search", Namespace: "default", UID: "request-uid", Generation: 1}, + } + request.Spec.Model = &nvidiacomv1beta2.ModelSpec{Name: "Qwen/Qwen3-0.6B"} + request.Spec.AISimulate = &nvidiacomv1beta2.AISimulateSpec{ + Image: "example.com/aisimulate:latest", + VersionOverride: "0.13.0", + } + if withHardware { + sku, budget := "h100_sxm", int32(4) + request.Spec.Hardware = &nvidiacomv1beta2.HardwareSpec{ + GPU: &nvidiacomv1beta2.GPUHardwareSpec{SKU: &sku, Budget: &budget}, + } + } + return request +} + +func dgdrV1beta2AcceptedCondition(t *testing.T, request *nvidiacomv1beta2.DynamoGraphDeploymentRequest) metav1.Condition { + t.Helper() + condition := meta.FindStatusCondition(request.Status.Conditions, DGDRAcceptedConditionV1beta2) + if condition == nil { + t.Fatal("no Accepted condition") + } + return *condition +} + +func TestV1beta2RequestControllerCreatesTheRunOfTheCurrentGeneration(t *testing.T) { + f := newDGDRV1beta2Fixture(t, dgdrV1beta2Request(true)) + f.reconcile() + + t.Log("One Run exists, owned by the Request, with the resolved and frozen input") + run := &nvidiacomv1beta2.DynamoGraphDeploymentRun{} + if err := f.client.Get(t.Context(), types.NamespacedName{Namespace: "default", Name: "search-1"}, run); err != nil { + t.Fatalf("get run: %v", err) + } + if !metav1.IsControlledBy(run, f.request()) { + t.Fatal("run is not controlled by the request") + } + if got := run.Spec.AISimulate.VersionOverride; got != "0.13.0" { + t.Fatalf("run versionOverride = %q, want the resolved 0.13.0", got) + } + parameters, err := aisimulate.DecodeParameters(run.Spec.AISimulate.Parameters) + if err != nil { + t.Fatalf("input of the run: %v", err) + } + searchSpace, _ := parameters["search_space"].(map[string]any) + if searchSpace["hardware_sku"] != "h100_sxm" || searchSpace["gpu_budget"] != float64(4) { + t.Fatalf("frozen search_space = %v, want the hardware of the request", searchSpace) + } + + t.Log("The status records the accepted generation and the active Run") + request := f.request() + if request.Status.ObservedGeneration != 1 { + t.Fatalf("observedGeneration = %d, want 1", request.Status.ObservedGeneration) + } + if request.Status.ActiveRunRef == nil || request.Status.ActiveRunRef.Name != "search-1" { + t.Fatalf("activeRunRef = %v, want search-1", request.Status.ActiveRunRef) + } + if condition := dgdrV1beta2AcceptedCondition(t, request); condition.Status != metav1.ConditionTrue { + t.Fatalf("Accepted = %v, want True", condition) + } +} + +func TestV1beta2RequestControllerIsIdempotent(t *testing.T) { + f := newDGDRV1beta2Fixture(t, dgdrV1beta2Request(true)) + f.reconcile() + f.reconcile() + f.reconcile() + if names := f.runNames(); len(names) != 1 || names[0] != "search-1" { + t.Fatalf("runs = %v, want only search-1", names) + } +} + +func TestV1beta2RequestControllerWaitsForTheActiveRunBeforeStartingTheNext(t *testing.T) { + f := newDGDRV1beta2Fixture(t, dgdrV1beta2Request(true)) + f.reconcile() + + t.Log("A spec change while search-1 runs creates nothing and says why") + f.bumpGeneration(2) + f.reconcile() + if names := f.runNames(); len(names) != 1 { + t.Fatalf("runs = %v, want only the active run", names) + } + request := f.request() + if request.Status.ObservedGeneration != 1 { + t.Fatalf("observedGeneration = %d, want 1 until generation 2 is accepted", request.Status.ObservedGeneration) + } + condition := dgdrV1beta2AcceptedCondition(t, request) + if condition.Status != metav1.ConditionFalse || condition.Reason != dgdrReasonWaitingForActiveRun { + t.Fatalf("condition = %v, want False/%s", condition, dgdrReasonWaitingForActiveRun) + } + + t.Log("Once search-1 is completed, search-2 is created and search-1 moves to the history") + f.finish("search-1", "Succeeded") + f.reconcile() + request = f.request() + if request.Status.ObservedGeneration != 2 { + t.Fatalf("observedGeneration = %d, want 2", request.Status.ObservedGeneration) + } + if request.Status.ActiveRunRef == nil || request.Status.ActiveRunRef.Name != "search-2" { + t.Fatalf("activeRunRef = %v, want search-2", request.Status.ActiveRunRef) + } + if len(request.Status.RecentRunRefs) != 1 || request.Status.RecentRunRefs[0].Name != "search-1" { + t.Fatalf("recentRunRefs = %v, want [search-1]", request.Status.RecentRunRefs) + } +} + +func TestV1beta2RequestControllerCoalescesIntermediateGenerations(t *testing.T) { + f := newDGDRV1beta2Fixture(t, dgdrV1beta2Request(true)) + f.reconcile() // search-1 is active + + f.bumpGeneration(2) + f.reconcile() + f.bumpGeneration(3) + f.reconcile() + + // Completing search-1 admits the current level, generation 3. + f.finish("search-1", "Succeeded") + f.reconcile() + + // Generation 2 was an intermediate desired state and is never materialized. + names := f.runNames() + if len(names) != 2 { + t.Fatalf("runs = %v, want search-1 and search-3", names) + } + for _, name := range names { + if name != "search-1" && name != "search-3" { + t.Fatalf("runs = %v, want search-1 and search-3 only", names) + } + } +} + +func TestV1beta2RequestControllerIgnoresAPhaseWithoutTheCompletedCondition(t *testing.T) { + f := newDGDRV1beta2Fixture(t, dgdrV1beta2Request(true)) + f.reconcile() + f.bumpGeneration(2) + + f.setPhaseOnly("search-1", "Succeeded") + f.reconcile() + + if names := f.runNames(); len(names) != 1 { + t.Fatalf("runs = %v, want the next Run held back until Completed is published", names) + } + if condition := dgdrV1beta2AcceptedCondition(t, f.request()); condition.Reason != dgdrReasonWaitingForActiveRun { + t.Fatalf("condition = %v, want reason %s", condition, dgdrReasonWaitingForActiveRun) + } +} + +func TestV1beta2RequestControllerReleasesOnCompletedWithoutATerminalPhase(t *testing.T) { + f := newDGDRV1beta2Fixture(t, dgdrV1beta2Request(true)) + f.reconcile() + f.bumpGeneration(2) + + f.setCompletedOnly("search-1") // the phase is still empty + f.reconcile() + + if names := f.runNames(); len(names) != 2 { + t.Fatalf("runs = %v, want search-2 started once Completed is True", names) + } +} + +func TestV1beta2RequestControllerCountsAnUnlabeledOwnedRunAsActive(t *testing.T) { + request := dgdrV1beta2Request(true) + active := dgdrV1beta2OwnedRun(request, 1, false) // correct controller OwnerReference, no labels + f := newDGDRV1beta2Fixture(t, request, active) + f.bumpGeneration(2) + + f.reconcile() + + if names := f.runNames(); len(names) != 1 || names[0] != "search-1" { + t.Fatalf("runs = %v, want no second active Run next to search-1", names) + } + if ref := f.request().Status.ActiveRunRef; ref == nil || ref.Name != "search-1" { + t.Fatalf("activeRunRef = %v, want search-1", ref) + } +} + +func TestV1beta2RequestControllerCreatesNoRunWithoutHardware(t *testing.T) { + f := newDGDRV1beta2Fixture(t, dgdrV1beta2Request(false)) + f.reconcile() + if names := f.runNames(); len(names) != 0 { + t.Fatalf("runs = %v, want none", names) + } + request := f.request() + condition := dgdrV1beta2AcceptedCondition(t, request) + if condition.Status != metav1.ConditionFalse || condition.Reason != dgdrReasonInvalidSpec { + t.Fatalf("condition = %v, want False/%s", condition, dgdrReasonInvalidSpec) + } + if request.Status.ObservedGeneration != 0 { + t.Fatalf("observedGeneration = %d, want 0 for a rejected generation", request.Status.ObservedGeneration) + } +} + +func TestV1beta2RequestControllerLeavesForeignCarriersAlone(t *testing.T) { + request := dgdrV1beta2Request(true) + carrier := request.DeepCopy() + carrier.Spec.V1Beta1 = &apiextensionsJSONForTest + f := newDGDRV1beta2Fixture(t, carrier) + f.reconcile() + if names := f.runNames(); len(names) != 0 { + t.Fatalf("runs = %v, want none for a v1beta1 carrier", names) + } + if got := f.request().Status.ObservedGeneration; got != 0 { + t.Fatalf("observedGeneration = %d, want the carrier's status untouched", got) + } +} + +func dgdrV1beta2RunFixtures(request *nvidiacomv1beta2.DynamoGraphDeploymentRequest, count int) []client.Object { + objects := []client.Object{request} + for generation := 1; generation <= count; generation++ { + objects = append(objects, dgdrV1beta2OwnedRun(request, generation, true)) + } + return objects +} + +func TestV1beta2RequestControllerRetainsFiveTerminalRuns(t *testing.T) { + request := dgdrV1beta2Request(true) + request.Generation = 8 + f := newDGDRV1beta2Fixture(t, dgdrV1beta2RunFixtures(request, 7)...) + f.reconcile() + + t.Log("The two oldest terminal runs are deleted and the newest five are listed newest first") + assertRetained(t, f) +} + +func assertRetained(t *testing.T, f *dgdrV1beta2Fixture) { + t.Helper() + for _, name := range []string{"search-1", "search-2"} { + err := f.client.Get(t.Context(), types.NamespacedName{Namespace: "default", Name: name}, &nvidiacomv1beta2.DynamoGraphDeploymentRun{}) + if !apierrors.IsNotFound(err) { + t.Fatalf("run %s: err = %v, want it deleted", name, err) + } + } + refs := f.request().Status.RecentRunRefs + want := []string{"search-7", "search-6", "search-5", "search-4", "search-3"} + if len(refs) != len(want) { + t.Fatalf("recentRunRefs = %v, want %v", refs, want) + } + for i := range want { + if refs[i].Name != want[i] { + t.Fatalf("recentRunRefs = %v, want %v", refs, want) + } + } +} + +func TestV1beta2RequestControllerRetentionIgnoresTheGenerationAnnotation(t *testing.T) { + request := dgdrV1beta2Request(true) + request.Generation = 8 + objects := dgdrV1beta2RunFixtures(request, 7) + for i, object := range objects[1:] { + if i%2 == 0 { + delete(object.GetAnnotations(), dgdrV1beta2GenerationAnnotation) // missing + } else { + object.GetAnnotations()[dgdrV1beta2GenerationAnnotation] = "garbage" // malformed + } + } + f := newDGDRV1beta2Fixture(t, objects...) + f.reconcile() + + assertRetained(t, f) +} + +func TestV1beta2RequestControllerRejectsAControlledRunWithoutAGenerationInItsName(t *testing.T) { + request := dgdrV1beta2Request(true) + odd := dgdrV1beta2OwnedRun(request, 1, true) + odd.Name = "unrelated" + f := newDGDRV1beta2Fixture(t, request, odd) + + if err := f.tryReconcile(); err == nil || !strings.Contains(err.Error(), "unrelated") { + t.Fatalf("err = %v, want an error naming the run", err) + } + if names := f.runNames(); len(names) != 1 { + t.Fatalf("runs = %v, want the odd run left alone and none created", names) + } +} + +func TestV1beta2RequestControllerPublishesStatusBeforeDeletingRetiredRuns(t *testing.T) { + request := dgdrV1beta2Request(true) + request.Generation = 8 + failPatch := true + f := newDGDRV1beta2FixtureWithInterceptor(t, interceptor.Funcs{ + SubResourcePatch: func( + ctx context.Context, c client.Client, subResource string, obj client.Object, + patch client.Patch, opts ...client.SubResourcePatchOption, + ) error { + if failPatch { + return errors.New("injected status patch failure") + } + return c.SubResource(subResource).Patch(ctx, obj, patch, opts...) + }, + }, dgdrV1beta2RunFixtures(request, 7)...) + + if err := f.tryReconcile(); err == nil { + t.Fatal("reconcile succeeded although the status patch failed") + } + // search-8 was created; the two oldest terminal Runs must still exist because the status did not + // move on. + if names := f.runNames(); len(names) != 8 { + t.Fatalf("runs = %v, want all eight: nothing may be deleted before the status is published", names) + } + + failPatch = false + f.reconcile() + assertRetained(t, f) +} + +func dgdrV1beta2OwnedRun( + request *nvidiacomv1beta2.DynamoGraphDeploymentRequest, + generation int, + terminal bool, +) *nvidiacomv1beta2.DynamoGraphDeploymentRun { + run := &nvidiacomv1beta2.DynamoGraphDeploymentRun{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%s-%d", request.Name, generation), + Namespace: request.Namespace, + Annotations: map[string]string{dgdrV1beta2GenerationAnnotation: fmt.Sprint(generation)}, + OwnerReferences: []metav1.OwnerReference{{ + APIVersion: nvidiacomv1beta2.GroupVersion.String(), + Kind: "DynamoGraphDeploymentRequest", + Name: request.Name, + UID: request.UID, + Controller: boolPointerForTest(true), + }}, + }, + } + if terminal { + run.Status.Phase = nvidiacomv1beta2.DGDRRunPhase("Succeeded") + setCompletedForTest(&run.Status) + } + return run +} + +func boolPointerForTest(value bool) *bool { return &value } + +// apiextensionsJSONForTest is a non-empty v1beta1 carrier payload. +var apiextensionsJSONForTest = apiextensionsv1.JSON{Raw: []byte(`{"model":"carrier"}`)} + +func TestV1beta2RequestControllerKeepsTheReadableRunNameWhenItFits(t *testing.T) { + request := dgdrV1beta2Request(true) + request.Name = strings.Repeat("a", 250) + request.Generation = 3 + if got, want := dgdrV1beta2RunName(request), request.Name+"-3"; got != want { + t.Fatalf("run name = %q, want %q", got, want) + } +} + +func TestV1beta2RequestControllerDerivesDistinctValidNamesForOverlengthRequests(t *testing.T) { + a := dgdrV1beta2Request(true) + a.Name = strings.Repeat("a", 252) + b := dgdrV1beta2Request(true) + b.Name = strings.Repeat("a", 251) + "b" // the same retained prefix as a + nameA, nameB := dgdrV1beta2RunName(a), dgdrV1beta2RunName(b) + for _, name := range []string{nameA, nameB} { + if len(name) > dgdrV1beta2MaxRunNameLength || !strings.HasSuffix(name, "-1") { + t.Fatalf("run name %q (%d characters) is not a valid name with the generation suffix", name, len(name)) + } + } + if nameA == nameB { + t.Fatalf("two different Request names produced the same Run name %q", nameA) + } + + t.Log("The name maps back to its generation for its own Request only") + run := &nvidiacomv1beta2.DynamoGraphDeploymentRun{ObjectMeta: metav1.ObjectMeta{Name: nameA}} + if generation, err := dgdrV1beta2RunGeneration(a.Name, run); err != nil || generation != 1 { + t.Fatalf("generation = %d, err = %v, want 1", generation, err) + } + if _, err := dgdrV1beta2RunGeneration(b.Name, run); err == nil { + t.Fatal("a Run name derived from another Request was accepted") + } +} + +func TestV1beta2RequestControllerReconcilesAnOverlengthRequestToOneRun(t *testing.T) { + request := dgdrV1beta2Request(true) + request.Name = strings.Repeat("a", 252) + f := newDGDRV1beta2Fixture(t, request) + key := types.NamespacedName{Namespace: request.Namespace, Name: request.Name} + for range 3 { + if _, err := f.reconciler.Reconcile(t.Context(), ctrl.Request{NamespacedName: key}); err != nil { + t.Fatalf("reconcile: %v", err) + } + } + want := dgdrV1beta2RunName(request) + if names := f.runNames(); len(names) != 1 || names[0] != want { + t.Fatalf("runs = %v, want only %q", names, want) + } + live := &nvidiacomv1beta2.DynamoGraphDeploymentRequest{} + if err := f.client.Get(t.Context(), key, live); err != nil { + t.Fatalf("get request: %v", err) + } + if live.Status.ActiveRunRef == nil || live.Status.ActiveRunRef.Name != want || live.Status.ObservedGeneration != 1 { + t.Fatalf("status = %+v, want the overlength Run active for generation 1", live.Status) + } +} + +func TestV1beta2RequestControllerDoesNotAdoptARunOfAnotherOwner(t *testing.T) { + stale := &nvidiacomv1beta2.DynamoGraphDeploymentRun{ + ObjectMeta: metav1.ObjectMeta{ + Name: "search-1", + Namespace: "default", + OwnerReferences: []metav1.OwnerReference{{ + APIVersion: nvidiacomv1beta2.GroupVersion.String(), + Kind: "DynamoGraphDeploymentRequest", + Name: "search", + UID: "old-uid", + Controller: boolPointerForTest(true), + }}, + }, + } + f := newDGDRV1beta2Fixture(t, dgdrV1beta2Request(true), stale) + + result, err := f.reconciler.Reconcile(t.Context(), ctrl.Request{ + NamespacedName: types.NamespacedName{Namespace: "default", Name: "search"}, + }) + if err != nil { + t.Fatalf("reconcile: %v", err) + } + if result.RequeueAfter <= 0 { + t.Fatal("no retry was scheduled for the Run name held by another owner") + } + request := f.request() + if condition := dgdrV1beta2AcceptedCondition(t, request); condition.Reason != dgdrReasonRunNameConflict { + t.Fatalf("condition = %v, want reason %s", condition, dgdrReasonRunNameConflict) + } + if request.Status.ActiveRunRef != nil { + t.Fatalf("activeRunRef = %v, want none: the old Run is not ours", request.Status.ActiveRunRef) + } + if request.Status.ObservedGeneration != 0 { + t.Fatalf("observedGeneration = %d, want 0", request.Status.ObservedGeneration) + } +} + +func TestV1beta2RequestControllerKeepsNullDeletedKeysOutOfTheRunInput(t *testing.T) { + request := dgdrV1beta2Request(true) + // The Request removes a typed value (workload isl) and a derived default (request_count) with null. + const spec = `{"model":{"name":"Qwen/Qwen3-0.6B"},"hardware":{"gpu":{"sku":"h100_sxm","budget":4}},` + + `"workload":{"static":{"isl":1024}},` + + `"aisimulate":{"image":"example.com/aisimulate:latest","versionOverride":"0.13.0",` + + `"parameters":{"workload":{"isl":null,"request_count":null}}}}` + request.Spec = nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec{} + if err := json.Unmarshal([]byte(spec), &request.Spec); err != nil { + t.Fatalf("decode spec: %v", err) + } + f := newDGDRV1beta2Fixture(t, request) + + run, err := f.reconciler.buildRun(request) + if err != nil { + t.Fatalf("build run: %v", err) + } + input, err := aisimulate.DecodeParameters(run.Spec.AISimulate.Parameters) + if err != nil { + t.Fatalf("input of the run: %v", err) + } + workload, _ := input["workload"].(map[string]any) + for _, key := range []string{"isl", "request_count"} { + if value, found := workload[key]; found { + t.Errorf("workload.%s = %v in the Run input, want it absent: the deletion must stay", key, value) + } + } +} diff --git a/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller.go b/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller.go new file mode 100644 index 000000000000..d1aa832f20de --- /dev/null +++ b/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller.go @@ -0,0 +1,675 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package controller + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "math" + "path" + "strconv" + "strings" + + configv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/config/v1alpha1" + nvidiacomv1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" + nvidiacomv1beta2 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta2" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/controller/aisimulate" + commoncontroller "github.com/ai-dynamo/dynamo/deploy/operator/internal/controller_common" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/dgdrrunpublisher" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/gpu" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/versionoverride" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + apiequality "k8s.io/apimachinery/pkg/api/equality" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/types" + "k8s.io/utils/ptr" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/yaml" +) + +const ( + dgdrRunJobControllerUIDIndex = ".metadata.dgdrRunControllerUID" + dgdrRunMaxJobNameLength = 63 + + dgdrRunInputKey = "input.yaml" + dgdrRunInputMountPath = "/etc/aisimulate" + dgdrRunInputPath = dgdrRunInputMountPath + "/" + dgdrRunInputKey + dgdrRunSnapshotPath = "/var/run/aisimulate/output" + dgdrRunServiceAccountMountPath = "/var/run/secrets/kubernetes.io/serviceaccount" + dgdrRunPublisherBinary = "/dgdr-run-publisher" + dgdrRunSweeperContainer = "sweeper" + dgdrRunPublisherContainer = "publisher" + dgdrRunInputVolume = "input" + dgdrRunSnapshotVolume = "snapshot" + dgdrRunServiceAccountVolume = "publisher-kube-api" + dgdrRunGPUsPerNodeAnnotation = "nvidia.com/dgdr-gpus-per-node" + dgdrRunSnapshotPathAnnotation = "nvidia.com/dgdr-snapshot-path" + dgdrRunResourceComponent = "aisimulate" + dgdrRunServiceAccountTokenTTL = int64(3600) + dgdrRunReasonPending = "JobPending" + dgdrRunReasonRunning = "JobRunning" + dgdrRunReasonSucceeded = "SearchSucceeded" + dgdrRunReasonSearchFailed = "SearchFailed" + dgdrRunReasonProtocolViolation = "ProtocolViolation" + dgdrRunReasonPublicationFailed = "PublicationFailed" + dgdrRunReasonJobFailed = "JobFailed" + dgdrRunMessagePending = "AISimulate Job is waiting to start" + dgdrRunMessageRunning = "AISimulate Job is running" + dgdrRunMessageSucceeded = "AISimulate search completed successfully" +) + +type dgdrRunHardwareDiscoverer interface { + Discover(context.Context, nvidiacomv1beta1.GPUSKUType) (*gpu.GPUInfo, error) +} + +// DynamoGraphDeploymentRunV1beta2Reconciler executes one immutable Run as an +// AISimulate Job. The publisher sidecar owns progress and candidate publication; +// this controller owns the Job reference, execution phase, timestamps and Completed. +type DynamoGraphDeploymentRunV1beta2Reconciler struct { + client.Client + Config *configv1alpha1.OperatorConfiguration + RuntimeConfig *commoncontroller.RuntimeConfig + Hardware dgdrRunHardwareDiscoverer + DynamoVersion string + OperatorImage string + OperatorImagePullPolicy corev1.PullPolicy +} + +// +kubebuilder:rbac:groups=nvidia.com,resources=dynamographdeploymentruns,verbs=get;list;watch +// +kubebuilder:rbac:groups=nvidia.com,resources=dynamographdeploymentruns/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=batch,resources=jobs,verbs=get;list;watch;create +// +kubebuilder:rbac:groups="",resources=pods,verbs=get;list;watch +// +kubebuilder:rbac:groups="",resources=nodes,verbs=get;list;watch +// +kubebuilder:rbac:groups="",resources=configmaps;serviceaccounts,verbs=get;list;watch;create;update;patch +// +kubebuilder:rbac:groups=rbac.authorization.k8s.io,resources=roles;rolebindings,verbs=get;list;watch;create;update;patch + +func (r *DynamoGraphDeploymentRunV1beta2Reconciler) Reconcile( + ctx context.Context, + req ctrl.Request, +) (ctrl.Result, error) { + run := &nvidiacomv1beta2.DynamoGraphDeploymentRun{} + if err := r.Get(ctx, req.NamespacedName, run); err != nil { + return ctrl.Result{}, client.IgnoreNotFound(err) + } + if run.DeletionTimestamp != nil || meta.IsStatusConditionTrue(run.Status.Conditions, nvidiacomv1beta2.DGDRRunConditionCompleted) { + return ctrl.Result{}, nil + } + if run.Spec.AISimulate == nil || run.Spec.AISimulate.Image == "" { + return ctrl.Result{}, fmt.Errorf("Run %q has no spec.aisimulate.image", run.Name) + } + + jobs, err := r.ownedJobs(ctx, run) + if err != nil { + return ctrl.Result{}, err + } + if len(jobs) > 1 { + return ctrl.Result{}, fmt.Errorf("Run %q controls %d Jobs; expected at most one", run.Name, len(jobs)) + } + + var job *batchv1.Job + if len(jobs) == 1 { + job = &jobs[0] + } + resourceName := dgdrRunResourceName(run.Name) + if err := r.reconcilePublisherAccess(ctx, run, resourceName); err != nil { + return ctrl.Result{}, err + } + execution, err := r.reconcileInput(ctx, run, job, resourceName) + if err != nil { + return ctrl.Result{}, err + } + + if job == nil { + job = r.buildJob(run, resourceName, execution) + if err := controllerutil.SetControllerReference(run, job, r.Scheme()); err != nil { + return ctrl.Result{}, fmt.Errorf("set Job owner: %w", err) + } + if err := r.Create(ctx, job); err != nil { + // A cache miss followed by AlreadyExists is expected to heal on the next + // reconciliation once the informer observes the Job. Never bypass the cache. + return ctrl.Result{}, fmt.Errorf("create Job %q: %w", job.Name, err) + } + } + + return ctrl.Result{}, r.reconcileRunStatus(ctx, run, job) +} + +func (r *DynamoGraphDeploymentRunV1beta2Reconciler) ownedJobs( + ctx context.Context, + run *nvidiacomv1beta2.DynamoGraphDeploymentRun, +) ([]batchv1.Job, error) { + list := &batchv1.JobList{} + if err := r.List(ctx, list, client.InNamespace(run.Namespace), + client.MatchingFields{dgdrRunJobControllerUIDIndex: string(run.UID)}); err != nil { + return nil, fmt.Errorf("list Jobs of Run %q: %w", run.Name, err) + } + owned := make([]batchv1.Job, 0, len(list.Items)) + for i := range list.Items { + if metav1.IsControlledBy(&list.Items[i], run) { + owned = append(owned, list.Items[i]) + } + } + return owned, nil +} + +func (r *DynamoGraphDeploymentRunV1beta2Reconciler) reconcilePublisherAccess( + ctx context.Context, + run *nvidiacomv1beta2.DynamoGraphDeploymentRun, + name string, +) error { + sa := &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: run.Namespace}} + if _, err := controllerutil.CreateOrUpdate(ctx, r.Client, sa, func() error { + return controllerutil.SetControllerReference(run, sa, r.Scheme()) + }); err != nil { + return fmt.Errorf("reconcile ServiceAccount %q: %w", name, err) + } + + role := &rbacv1.Role{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: run.Namespace}} + if _, err := controllerutil.CreateOrUpdate(ctx, r.Client, role, func() error { + role.Rules = dgdrRunPublisherRules(run.Name) + return controllerutil.SetControllerReference(run, role, r.Scheme()) + }); err != nil { + return fmt.Errorf("reconcile Role %q: %w", name, err) + } + + binding := &rbacv1.RoleBinding{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: run.Namespace}} + if _, err := controllerutil.CreateOrUpdate(ctx, r.Client, binding, func() error { + binding.RoleRef = rbacv1.RoleRef{APIGroup: rbacv1.GroupName, Kind: "Role", Name: name} + binding.Subjects = []rbacv1.Subject{{Kind: "ServiceAccount", Name: name, Namespace: run.Namespace}} + return controllerutil.SetControllerReference(run, binding, r.Scheme()) + }); err != nil { + return fmt.Errorf("reconcile RoleBinding %q: %w", name, err) + } + return nil +} + +func dgdrRunPublisherRules(runName string) []rbacv1.PolicyRule { + return []rbacv1.PolicyRule{ + { + APIGroups: []string{nvidiacomv1beta2.GroupVersion.Group}, + Resources: []string{"dynamographdeploymentruns"}, + ResourceNames: []string{runName}, + Verbs: []string{"get", "list", "watch"}, + }, + { + APIGroups: []string{nvidiacomv1beta2.GroupVersion.Group}, + Resources: []string{"dynamographdeploymentruns/status"}, + ResourceNames: []string{runName}, + Verbs: []string{"get", "update", "patch"}, + }, + { + APIGroups: []string{nvidiacomv1beta2.GroupVersion.Group}, + Resources: []string{"dynamographdeploymentcandidates"}, + Verbs: []string{"get", "list", "watch", "create", "patch", "delete"}, + }, + { + APIGroups: []string{nvidiacomv1beta2.GroupVersion.Group}, + Resources: []string{"dynamographdeploymentcandidates/status"}, + Verbs: []string{"get", "update", "patch"}, + }, + { + APIGroups: []string{""}, + Resources: []string{"pods"}, + Verbs: []string{"get", "list", "watch"}, + }, + } +} + +func (r *DynamoGraphDeploymentRunV1beta2Reconciler) reconcileInput( + ctx context.Context, + run *nvidiacomv1beta2.DynamoGraphDeploymentRun, + job *batchv1.Job, + name string, +) (dgdrRunExecution, error) { + configMap := &corev1.ConfigMap{} + key := types.NamespacedName{Namespace: run.Namespace, Name: name} + err := r.Get(ctx, key, configMap) + if err == nil { + if !metav1.IsControlledBy(configMap, run) { + return dgdrRunExecution{}, fmt.Errorf("ConfigMap %q is not controlled by Run %q", name, run.Name) + } + gpusPerNode, parseErr := positiveAnnotation(configMap.Annotations, dgdrRunGPUsPerNodeAnnotation) + if parseErr != nil { + return dgdrRunExecution{}, fmt.Errorf("read ConfigMap %q: %w", name, parseErr) + } + execution := dgdrRunExecution{ + GPUsPerNode: gpusPerNode, + SnapshotPath: annotationOrDefault(configMap.Annotations, dgdrRunSnapshotPathAnnotation, dgdrRunSnapshotPath), + } + if job != nil { + jobGPUs, jobErr := positiveAnnotation(job.Annotations, dgdrRunGPUsPerNodeAnnotation) + if jobErr != nil { + return dgdrRunExecution{}, fmt.Errorf("read Job %q: %w", job.Name, jobErr) + } + if jobGPUs != gpusPerNode { + return dgdrRunExecution{}, fmt.Errorf("Job %q and ConfigMap %q disagree about GPUs per node", job.Name, name) + } + jobSnapshotPath := annotationOrDefault(job.Annotations, dgdrRunSnapshotPathAnnotation, dgdrRunSnapshotPath) + if jobSnapshotPath != execution.SnapshotPath { + return dgdrRunExecution{}, fmt.Errorf("Job %q and ConfigMap %q disagree about the snapshot path", job.Name, name) + } + } + return execution, nil + } + if !apierrors.IsNotFound(err) { + return dgdrRunExecution{}, fmt.Errorf("get ConfigMap %q: %w", name, err) + } + + execution := dgdrRunExecution{SnapshotPath: dgdrRunSnapshotPath} + if job != nil { + execution.GPUsPerNode, err = positiveAnnotation(job.Annotations, dgdrRunGPUsPerNodeAnnotation) + if err != nil { + return dgdrRunExecution{}, fmt.Errorf("read Job %q: %w", job.Name, err) + } + execution.SnapshotPath = annotationOrDefault(job.Annotations, dgdrRunSnapshotPathAnnotation, dgdrRunSnapshotPath) + } + input, err := aisimulate.DecodeParameters(run.Spec.AISimulate.Parameters) + if err != nil { + return dgdrRunExecution{}, err + } + if execution.GPUsPerNode == 0 { + if configured, found := nestedPositiveInt(input, "dgdr_run", "num_gpus_per_node"); found { + execution.GPUsPerNode = configured + } else { + execution.GPUsPerNode, err = r.discoverGPUsPerNode(ctx, input) + if err != nil { + return dgdrRunExecution{}, err + } + } + } + contents, snapshotPath, err := dgdrRunInput(run, input, r.DynamoVersion, execution.GPUsPerNode) + if err != nil { + return dgdrRunExecution{}, err + } + if job == nil { + execution.SnapshotPath = snapshotPath + } + + configMap = &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: run.Namespace, + Annotations: map[string]string{ + dgdrRunGPUsPerNodeAnnotation: strconv.Itoa(execution.GPUsPerNode), + dgdrRunSnapshotPathAnnotation: execution.SnapshotPath, + }, + }, + Immutable: ptr.To(true), + Data: map[string]string{dgdrRunInputKey: string(contents)}, + } + if err := controllerutil.SetControllerReference(run, configMap, r.Scheme()); err != nil { + return dgdrRunExecution{}, fmt.Errorf("set ConfigMap owner: %w", err) + } + if err := r.Create(ctx, configMap); err != nil { + return dgdrRunExecution{}, fmt.Errorf("create ConfigMap %q: %w", name, err) + } + return execution, nil +} + +func (r *DynamoGraphDeploymentRunV1beta2Reconciler) discoverGPUsPerNode( + ctx context.Context, + input map[string]any, +) (int, error) { + if r.Hardware == nil { + return 0, errors.New("discover GPUs per node: hardware discoverer is not configured") + } + sku, _, err := unstructured.NestedString(input, "search_space", "hardware_sku") + if err != nil { + return 0, fmt.Errorf("read search_space.hardware_sku: %w", err) + } + info, err := r.Hardware.Discover(ctx, nvidiacomv1beta1.GPUSKUType(sku)) + if err != nil { + return 0, fmt.Errorf("discover GPUs per node: %w", err) + } + if info == nil || info.GPUsPerNode < 1 { + return 0, errors.New("discover GPUs per node: discovery returned no positive GPU count") + } + return info.GPUsPerNode, nil +} + +func dgdrRunInput( + run *nvidiacomv1beta2.DynamoGraphDeploymentRun, + parameters map[string]any, + dynamoVersion string, + gpusPerNode int, +) ([]byte, string, error) { + if run.Spec.Recommendation == nil || run.Spec.Recommendation.MaxCandidates < 1 { + return nil, "", errors.New("spec.recommendation.maxCandidates must be defaulted before the Run is created") + } + output := map[string]any{ + "name": run.Name, + "snapshot_dir": dgdrRunSnapshotPath, + "namespace": run.Namespace, + "runtime_image": run.Spec.AISimulate.Image, + "derive_runtime_images": true, + "num_gpus_per_node": int64(gpusPerNode), + "max_candidates": int64(run.Spec.Recommendation.MaxCandidates), + } + if _, err := versionoverride.ParseImageVersion(run.Spec.AISimulate.Image); err != nil { + if _, versionErr := versionoverride.Parse(dynamoVersion); versionErr != nil { + return nil, "", fmt.Errorf("resolve Dynamo version for planner image %q: %w", run.Spec.AISimulate.Image, versionErr) + } + output["runtime_version_override"] = dynamoVersion + } + underlay := map[string]any{"dgdr_run": output} + input := aisimulate.MergeParameters(underlay, parameters) + snapshotPath := dgdrRunSnapshotPath + if configured, found, err := unstructured.NestedString(input, "dgdr_run", "snapshot_dir"); err == nil && found && path.IsAbs(configured) { + snapshotPath = path.Clean(configured) + } + contents, err := yaml.Marshal(input) + if err != nil { + return nil, "", fmt.Errorf("encode AISimulate input: %w", err) + } + return contents, snapshotPath, nil +} + +type dgdrRunExecution struct { + GPUsPerNode int + SnapshotPath string +} + +func nestedPositiveInt(input map[string]any, fields ...string) (int, bool) { + value, found, err := unstructured.NestedFieldNoCopy(input, fields...) + if err != nil || !found { + return 0, false + } + var number float64 + switch typed := value.(type) { + case int64: + number = float64(typed) + case float64: + number = typed + default: + return 0, false + } + if number < 1 || number > math.MaxInt || math.Trunc(number) != number { + return 0, false + } + return int(number), true +} + +func positiveAnnotation(annotations map[string]string, key string) (int, error) { + value := annotations[key] + parsed, err := strconv.Atoi(value) + if err != nil || parsed < 1 { + return 0, fmt.Errorf("annotation %s=%q is not a positive integer", key, value) + } + return parsed, nil +} + +func annotationOrDefault(annotations map[string]string, key, defaultValue string) string { + if value := annotations[key]; value != "" { + return value + } + return defaultValue +} + +func (r *DynamoGraphDeploymentRunV1beta2Reconciler) buildJob( + run *nvidiacomv1beta2.DynamoGraphDeploymentRun, + name string, + execution dgdrRunExecution, +) *batchv1.Job { + pullPolicy := r.OperatorImagePullPolicy + if pullPolicy == "" { + pullPolicy = corev1.PullIfNotPresent + } + job := &batchv1.Job{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: run.Namespace, + Annotations: map[string]string{ + dgdrRunGPUsPerNodeAnnotation: strconv.Itoa(execution.GPUsPerNode), + dgdrRunSnapshotPathAnnotation: execution.SnapshotPath, + }, + }, + Spec: batchv1.JobSpec{ + BackoffLimit: ptr.To[int32](0), + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + ServiceAccountName: name, + AutomountServiceAccountToken: ptr.To(false), + RestartPolicy: corev1.RestartPolicyNever, + SecurityContext: &corev1.PodSecurityContext{FSGroup: ptr.To[int64](65532)}, + Containers: []corev1.Container{ + { + Name: dgdrRunSweeperContainer, + Image: run.Spec.AISimulate.Image, + ImagePullPolicy: corev1.PullIfNotPresent, + Command: []string{"aisimulate"}, + Args: []string{ + "recommend", "--stack", "dynamo", "--config", dgdrRunInputPath, + "--output", "dgdr_run", "--output-dir", execution.SnapshotPath, + }, + VolumeMounts: []corev1.VolumeMount{ + {Name: dgdrRunInputVolume, MountPath: dgdrRunInputMountPath, ReadOnly: true}, + {Name: dgdrRunSnapshotVolume, MountPath: execution.SnapshotPath}, + }, + }, + { + Name: dgdrRunPublisherContainer, + Image: r.OperatorImage, + ImagePullPolicy: pullPolicy, + Command: []string{dgdrRunPublisherBinary}, + Args: []string{ + "--snapshot-dir", execution.SnapshotPath, + "--run-name", run.Name, + "--run-uid", string(run.UID), + "--sweeper-container", dgdrRunSweeperContainer, + }, + Env: []corev1.EnvVar{ + {Name: "POD_NAMESPACE", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.namespace"}}}, + {Name: "POD_NAME", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.name"}}}, + }, + VolumeMounts: []corev1.VolumeMount{ + {Name: dgdrRunSnapshotVolume, MountPath: execution.SnapshotPath}, + {Name: dgdrRunServiceAccountVolume, MountPath: dgdrRunServiceAccountMountPath, ReadOnly: true}, + }, + }, + }, + Volumes: []corev1.Volume{ + { + Name: dgdrRunInputVolume, + VolumeSource: corev1.VolumeSource{ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{Name: name}, + }}, + }, + {Name: dgdrRunSnapshotVolume, VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}}, + dgdrRunPublisherKubeAPIVolume(), + }, + }, + }, + }, + } + return job +} + +func dgdrRunPublisherKubeAPIVolume() corev1.Volume { + return corev1.Volume{ + Name: dgdrRunServiceAccountVolume, + VolumeSource: corev1.VolumeSource{Projected: &corev1.ProjectedVolumeSource{ + Sources: []corev1.VolumeProjection{ + {ServiceAccountToken: &corev1.ServiceAccountTokenProjection{ + Path: "token", ExpirationSeconds: ptr.To(dgdrRunServiceAccountTokenTTL), + }}, + {ConfigMap: &corev1.ConfigMapProjection{ + LocalObjectReference: corev1.LocalObjectReference{Name: ConfigMapNameKubeRootCA}, + Items: []corev1.KeyToPath{{Key: "ca.crt", Path: "ca.crt"}}, + }}, + {DownwardAPI: &corev1.DownwardAPIProjection{Items: []corev1.DownwardAPIVolumeFile{{ + Path: "namespace", FieldRef: &corev1.ObjectFieldSelector{APIVersion: "v1", FieldPath: "metadata.namespace"}, + }}}}, + }, + }}, + } +} + +func (r *DynamoGraphDeploymentRunV1beta2Reconciler) reconcileRunStatus( + ctx context.Context, + run *nvidiacomv1beta2.DynamoGraphDeploymentRun, + job *batchv1.Job, +) error { + original := run.DeepCopy() + run.Status.JobRef = &nvidiacomv1beta2.ResourceReference{Name: job.Name, UID: job.UID} + run.Status.Phase = nvidiacomv1beta2.DGDRRunPhasePending + reason, message := dgdrRunReasonPending, dgdrRunMessagePending + completed := metav1.ConditionFalse + + if job.Status.StartTime != nil { + run.Status.StartTime = job.Status.StartTime.DeepCopy() + } + if condition := dgdrRunJobCondition(job, batchv1.JobComplete); condition != nil && condition.Status == corev1.ConditionTrue { + run.Status.Phase = nvidiacomv1beta2.DGDRRunPhaseSucceeded + reason, message = dgdrRunReasonSucceeded, dgdrRunMessageSucceeded + completed = metav1.ConditionTrue + run.Status.CompletionTime = dgdrRunCompletionTime(job) + } else if condition := dgdrRunJobCondition(job, batchv1.JobFailed); condition != nil && condition.Status == corev1.ConditionTrue { + run.Status.Phase = nvidiacomv1beta2.DGDRRunPhaseFailed + reason, message = r.dgdrRunFailure(ctx, job, condition) + completed = metav1.ConditionTrue + run.Status.CompletionTime = dgdrRunCompletionTime(job) + } else if job.Status.Active > 0 || job.Status.StartTime != nil { + run.Status.Phase = nvidiacomv1beta2.DGDRRunPhaseRunning + reason, message = dgdrRunReasonRunning, dgdrRunMessageRunning + } + + meta.SetStatusCondition(&run.Status.Conditions, metav1.Condition{ + Type: nvidiacomv1beta2.DGDRRunConditionCompleted, + Status: completed, + Reason: reason, + Message: message, + ObservedGeneration: run.Generation, + }) + if apiequality.Semantic.DeepEqual(original.Status, run.Status) { + return nil + } + if err := r.Status().Patch(ctx, run, client.MergeFrom(original)); err != nil { + return fmt.Errorf("update Run %q status: %w", run.Name, err) + } + return nil +} + +func (r *DynamoGraphDeploymentRunV1beta2Reconciler) dgdrRunFailure( + ctx context.Context, + job *batchv1.Job, + condition *batchv1.JobCondition, +) (string, string) { + pods := &corev1.PodList{} + if err := r.List(ctx, pods, client.InNamespace(job.Namespace), client.MatchingLabels{batchv1.JobNameLabel: job.Name}); err != nil { + log.FromContext(ctx).Error(err, "Could not inspect failed AISimulate Job pods", "job", job.Name) + return dgdrRunReasonJobFailed, condition.Message + } + for i := range pods.Items { + pod := &pods.Items[i] + if !metav1.IsControlledBy(pod, job) { + continue + } + for _, status := range pod.Status.ContainerStatuses { + terminated := status.State.Terminated + if terminated == nil || terminated.ExitCode == 0 { + continue + } + message := terminated.Message + if message == "" { + message = terminated.Reason + } + if message == "" { + message = fmt.Sprintf("container %s exited with code %d", status.Name, terminated.ExitCode) + } + if status.Name == dgdrRunPublisherContainer { + switch int(terminated.ExitCode) { + case dgdrrunpublisher.ExitRunFailed: + return dgdrRunReasonSearchFailed, message + case dgdrrunpublisher.ExitProtocolViolation: + return dgdrRunReasonProtocolViolation, message + case dgdrrunpublisher.ExitReconcileFailed: + return dgdrRunReasonPublicationFailed, message + } + } + if status.Name == dgdrRunSweeperContainer { + return dgdrRunReasonSearchFailed, message + } + } + } + message := condition.Message + if message == "" { + message = "AISimulate Job failed" + } + return dgdrRunReasonJobFailed, message +} + +func dgdrRunCompletionTime(job *batchv1.Job) *metav1.Time { + if job.Status.CompletionTime != nil { + return job.Status.CompletionTime.DeepCopy() + } + for _, condition := range job.Status.Conditions { + if (condition.Type == batchv1.JobComplete || condition.Type == batchv1.JobFailed) && condition.Status == corev1.ConditionTrue { + return condition.LastTransitionTime.DeepCopy() + } + } + return nil +} + +func dgdrRunJobCondition(job *batchv1.Job, conditionType batchv1.JobConditionType) *batchv1.JobCondition { + for i := range job.Status.Conditions { + if job.Status.Conditions[i].Type == conditionType { + return &job.Status.Conditions[i] + } + } + return nil +} + +func dgdrRunResourceName(runName string) string { + suffix := "-" + dgdrRunResourceComponent + if len(runName)+len(suffix) <= dgdrRunMaxJobNameLength { + return runName + suffix + } + sum := sha256.Sum256([]byte(runName)) + hash := hex.EncodeToString(sum[:6]) + room := dgdrRunMaxJobNameLength - len(suffix) - len(hash) - 1 + prefix := strings.TrimRight(runName[:room], "-.") + return prefix + "-" + hash + suffix +} + +func (r *DynamoGraphDeploymentRunV1beta2Reconciler) SetupWithManager(mgr ctrl.Manager) error { + if r.OperatorImage == "" { + return errors.New("operator image is required for the DGDR Run publisher") + } + if err := mgr.GetFieldIndexer().IndexField( + context.Background(), &batchv1.Job{}, dgdrRunJobControllerUIDIndex, dgdrRunJobControllerUIDs, + ); err != nil { + return fmt.Errorf("index Jobs by Run controller UID: %w", err) + } + return ctrl.NewControllerManagedBy(mgr). + Named("dynamographdeploymentrun-v1beta2"). + For(&nvidiacomv1beta2.DynamoGraphDeploymentRun{}). + Owns(&batchv1.Job{}). + Owns(&corev1.ConfigMap{}). + Owns(&corev1.ServiceAccount{}). + Owns(&rbacv1.Role{}). + Owns(&rbacv1.RoleBinding{}). + WithEventFilter(commoncontroller.EphemeralDeploymentEventFilter(r.Config, r.RuntimeConfig)). + Complete(r) +} + +func dgdrRunJobControllerUIDs(obj client.Object) []string { + owner := metav1.GetControllerOf(obj) + if owner == nil || owner.Kind != "DynamoGraphDeploymentRun" || owner.APIVersion != nvidiacomv1beta2.GroupVersion.String() { + return nil + } + return []string{string(owner.UID)} +} diff --git a/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller_test.go b/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller_test.go new file mode 100644 index 000000000000..1cfc3dc4b6f4 --- /dev/null +++ b/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller_test.go @@ -0,0 +1,442 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package controller + +import ( + "context" + "encoding/json" + "reflect" + "strings" + "testing" + "time" + + nvidiacomv1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" + nvidiacomv1beta2 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta2" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/dgdrrunpublisher" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/gpu" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + "k8s.io/utils/ptr" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/yaml" +) + +type fixedRunHardware struct { + info gpu.GPUInfo + err error + calls []nvidiacomv1beta1.GPUSKUType +} + +func (f *fixedRunHardware) Discover(_ context.Context, sku nvidiacomv1beta1.GPUSKUType) (*gpu.GPUInfo, error) { + f.calls = append(f.calls, sku) + if f.err != nil { + return nil, f.err + } + return &f.info, nil +} + +type dgdrRunFixture struct { + t *testing.T + client client.Client + reconciler *DynamoGraphDeploymentRunV1beta2Reconciler +} + +func newDGDRRunFixture(t *testing.T, hardware dgdrRunHardwareDiscoverer, objects ...client.Object) *dgdrRunFixture { + t.Helper() + scheme := runtime.NewScheme() + if err := clientgoscheme.AddToScheme(scheme); err != nil { + t.Fatalf("add Kubernetes types: %v", err) + } + if err := nvidiacomv1beta2.AddToScheme(scheme); err != nil { + t.Fatalf("add v1beta2 types: %v", err) + } + c := fake.NewClientBuilder(). + WithScheme(scheme). + WithObjects(objects...). + WithIndex(&batchv1.Job{}, dgdrRunJobControllerUIDIndex, dgdrRunJobControllerUIDs). + WithStatusSubresource(&nvidiacomv1beta2.DynamoGraphDeploymentRun{}, &batchv1.Job{}, &corev1.Pod{}). + Build() + return &dgdrRunFixture{ + t: t, + client: c, + reconciler: &DynamoGraphDeploymentRunV1beta2Reconciler{ + Client: c, + Hardware: hardware, + DynamoVersion: "1.2.3", + OperatorImage: "example.com/dynamo-operator:1.2.3", + OperatorImagePullPolicy: corev1.PullAlways, + }, + } +} + +func (f *dgdrRunFixture) reconcile() { + f.t.Helper() + if _, err := f.reconciler.Reconcile(f.t.Context(), ctrl.Request{NamespacedName: types.NamespacedName{ + Namespace: "default", Name: "search-1", + }}); err != nil { + f.t.Fatalf("reconcile: %v", err) + } +} + +func (f *dgdrRunFixture) run() *nvidiacomv1beta2.DynamoGraphDeploymentRun { + f.t.Helper() + run := &nvidiacomv1beta2.DynamoGraphDeploymentRun{} + if err := f.client.Get(f.t.Context(), types.NamespacedName{Namespace: "default", Name: "search-1"}, run); err != nil { + f.t.Fatalf("get Run: %v", err) + } + return run +} + +func (f *dgdrRunFixture) job() *batchv1.Job { + f.t.Helper() + jobs := &batchv1.JobList{} + if err := f.client.List(f.t.Context(), jobs, client.InNamespace("default")); err != nil { + f.t.Fatalf("list Jobs: %v", err) + } + if len(jobs.Items) != 1 { + f.t.Fatalf("Jobs = %d, want 1", len(jobs.Items)) + } + return &jobs.Items[0] +} + +func dgdrRunForTest(t *testing.T, parameters map[string]any) *nvidiacomv1beta2.DynamoGraphDeploymentRun { + t.Helper() + raw, err := json.Marshal(parameters) + if err != nil { + t.Fatalf("marshal parameters: %v", err) + } + return &nvidiacomv1beta2.DynamoGraphDeploymentRun{ + ObjectMeta: metav1.ObjectMeta{Name: "search-1", Namespace: "default", UID: "run-uid", Generation: 1}, + Spec: nvidiacomv1beta2.DynamoGraphDeploymentRunSpec{NativeSearchInput: nvidiacomv1beta2.NativeSearchInput{ + Model: &nvidiacomv1beta2.ModelSpec{Name: "Qwen/Qwen3-0.6B"}, + AISimulate: &nvidiacomv1beta2.AISimulateSpec{Image: "example.com/dynamo-planner:1.2.3", Parameters: &runtime.RawExtension{Raw: raw}}, + Recommendation: &nvidiacomv1beta2.RecommendationSpec{MaxCandidates: 2}, + }}, + } +} + +func TestV1beta2RunControllerCreatesOneIsolatedAISimulateJob(t *testing.T) { + hardware := &fixedRunHardware{info: gpu.GPUInfo{GPUsPerNode: 8}} + run := dgdrRunForTest(t, map[string]any{ + "search_space": map[string]any{"hardware_sku": "h200_sxm", "gpu_budget": 16}, + "dgdr_run": map[string]any{ + "max_candidates": 7, + "runtime_image": "example.com/custom-vllm-runtime:1.2.3", + "name": "native-output-name", + "snapshot_dir": "/work/results", + }, + }) + f := newDGDRRunFixture(t, hardware, run) + + f.reconcile() + f.reconcile() + + job := f.job() + assertDGDRRunJobIsolation(t, job, run, f.reconciler.OperatorImage) + assertDGDRRunInput(t, f.client, job, hardware) + assertDGDRRunSupportResources(t, f.client, job, run) + assertDGDRRunPendingStatus(t, f.run(), job.Name) +} + +func assertDGDRRunJobIsolation( + t *testing.T, + job *batchv1.Job, + run *nvidiacomv1beta2.DynamoGraphDeploymentRun, + operatorImage string, +) { + t.Helper() + if !metav1.IsControlledBy(job, run) { + t.Fatal("Job is not controlled by its Run") + } + if got := job.Annotations[dgdrRunGPUsPerNodeAnnotation]; got != "8" { + t.Fatalf("GPUs-per-node annotation = %q, want 8", got) + } + if job.Spec.BackoffLimit == nil || *job.Spec.BackoffLimit != 0 { + t.Fatalf("backoffLimit = %v, want 0", job.Spec.BackoffLimit) + } + if job.Spec.Template.Spec.AutomountServiceAccountToken == nil || *job.Spec.Template.Spec.AutomountServiceAccountToken { + t.Fatal("the pod must disable automatic service-account token mounting") + } + sweeper := containerNamed(t, job.Spec.Template.Spec.Containers, dgdrRunSweeperContainer) + publisher := containerNamed(t, job.Spec.Template.Spec.Containers, dgdrRunPublisherContainer) + if sweeper.Image != run.Spec.AISimulate.Image { + t.Fatalf("sweeper image = %q, want %q", sweeper.Image, run.Spec.AISimulate.Image) + } + if want := []string{"aisimulate"}; !reflect.DeepEqual(sweeper.Command, want) { + t.Fatalf("sweeper command = %q, want %q", sweeper.Command, want) + } + if want := []string{ + "recommend", "--stack", "dynamo", "--config", dgdrRunInputPath, + "--output", "dgdr_run", "--output-dir", "/work/results", + }; !reflect.DeepEqual(sweeper.Args, want) { + t.Fatalf("sweeper args = %q, want %q", sweeper.Args, want) + } + if publisher.Image != operatorImage || publisher.ImagePullPolicy != corev1.PullAlways { + t.Fatalf("publisher image = %q/%s, want configured operator image/Always", publisher.Image, publisher.ImagePullPolicy) + } + if hasMount(sweeper.VolumeMounts, dgdrRunServiceAccountVolume) { + t.Fatal("sweeper unexpectedly receives the Kubernetes service-account token") + } + if !hasMount(publisher.VolumeMounts, dgdrRunServiceAccountVolume) { + t.Fatal("publisher does not receive its projected Kubernetes credentials") + } + if !hasMountAt(sweeper.VolumeMounts, dgdrRunSnapshotVolume, "/work/results") || + !hasMountAt(publisher.VolumeMounts, dgdrRunSnapshotVolume, "/work/results") { + t.Fatal("sweeper and publisher do not share the parameter-overridden snapshot directory") + } +} + +func assertDGDRRunInput(t *testing.T, c client.Client, job *batchv1.Job, hardware *fixedRunHardware) { + t.Helper() + configMap := &corev1.ConfigMap{} + if err := c.Get(t.Context(), client.ObjectKeyFromObject(job), configMap); err != nil { + t.Fatalf("get input ConfigMap: %v", err) + } + if configMap.Immutable == nil || !*configMap.Immutable { + t.Fatal("input ConfigMap is not immutable") + } + var input map[string]any + if err := yaml.Unmarshal([]byte(configMap.Data[dgdrRunInputKey]), &input); err != nil { + t.Fatalf("decode input ConfigMap: %v", err) + } + output := input["dgdr_run"].(map[string]any) + if output["name"] != "native-output-name" || output["snapshot_dir"] != "/work/results" { + t.Fatalf("native parameter overrides were not preserved: %v", output) + } + if output["max_candidates"] != float64(7) || output["runtime_image"] != "example.com/custom-vllm-runtime:1.2.3" { + t.Fatalf("native parameter overrides were not preserved: %v", output) + } + if output["derive_runtime_images"] != true { + t.Fatalf("derive_runtime_images = %v, want true for a backend search", output["derive_runtime_images"]) + } + if got := input["search_space"].(map[string]any)["hardware_sku"]; got != "h200_sxm" { + t.Fatalf("search_space.hardware_sku = %v, want h200_sxm", got) + } + if !reflect.DeepEqual(hardware.calls, []nvidiacomv1beta1.GPUSKUType{"h200_sxm"}) { + t.Fatalf("hardware discovery calls = %v, want one filtered call", hardware.calls) + } +} + +func TestDGDRRunInputResolvesRuntimeVersion(t *testing.T) { + tests := []struct { + name string + plannerImage string + dynamoVersion string + wantOverride string + wantErr bool + }{ + {name: "canonical planner tag", plannerImage: "example.com/dynamo-planner:1.7.0"}, + {name: "opaque CI tag", plannerImage: "example.com/dynamo-planner:git-sha", dynamoVersion: "1.7.0", wantOverride: "1.7.0"}, + {name: "opaque tag without canonical version", plannerImage: "example.com/dynamo-planner:git-sha", dynamoVersion: "1.7", wantErr: true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + run := dgdrRunForTest(t, map[string]any{}) + run.Spec.AISimulate.Image = tt.plannerImage + + contents, _, err := dgdrRunInput(run, map[string]any{}, tt.dynamoVersion, 8) + if (err != nil) != tt.wantErr { + t.Fatalf("lower input error = %v, wantErr %t", err, tt.wantErr) + } + if tt.wantErr { + return + } + input := map[string]any{} + if err := yaml.Unmarshal(contents, &input); err != nil { + t.Fatalf("decode input: %v", err) + } + output := input["dgdr_run"].(map[string]any) + got, found := output["runtime_version_override"] + if tt.wantOverride == "" && found { + t.Fatalf("dgdr_run.runtime_version_override = %v, want omitted", got) + } + if tt.wantOverride != "" && got != tt.wantOverride { + t.Fatalf("dgdr_run.runtime_version_override = %v, want %s", got, tt.wantOverride) + } + }) + } +} + +func assertDGDRRunSupportResources( + t *testing.T, + c client.Client, + job *batchv1.Job, + run *nvidiacomv1beta2.DynamoGraphDeploymentRun, +) { + t.Helper() + for _, object := range []client.Object{ + &corev1.ServiceAccount{}, &rbacv1.Role{}, &rbacv1.RoleBinding{}, + } { + if err := c.Get(t.Context(), client.ObjectKeyFromObject(job), object); err != nil { + t.Fatalf("get %T: %v", object, err) + } + if !metav1.IsControlledBy(object, run) { + t.Fatalf("%T is not controlled by the Run", object) + } + } +} + +func assertDGDRRunPendingStatus(t *testing.T, stored *nvidiacomv1beta2.DynamoGraphDeploymentRun, jobName string) { + t.Helper() + if stored.Status.Phase != nvidiacomv1beta2.DGDRRunPhasePending || stored.Status.JobRef == nil || stored.Status.JobRef.Name != jobName { + t.Fatalf("Run status = %+v, want Pending and JobRef %q", stored.Status, jobName) + } + condition := meta.FindStatusCondition(stored.Status.Conditions, nvidiacomv1beta2.DGDRRunConditionCompleted) + if condition == nil || condition.Status != metav1.ConditionFalse || condition.Reason != dgdrRunReasonPending { + t.Fatalf("Completed condition = %+v, want False/%s", condition, dgdrRunReasonPending) + } +} + +func TestV1beta2RunControllerOwnsOnlyLifecycleStatus(t *testing.T) { + hardware := &fixedRunHardware{info: gpu.GPUInfo{GPUsPerNode: 8}} + f := newDGDRRunFixture(t, hardware, dgdrRunForTest(t, map[string]any{ + "search_space": map[string]any{"hardware_sku": "h200_sxm", "gpu_budget": 8}, + })) + f.reconcile() + job := f.job() + + run := f.run() + progressTime := metav1.NewTime(time.Date(2026, 10, 23, 10, 0, 0, 0, time.UTC)) + run.Status.Progress = &nvidiacomv1beta2.DGDRRunProgress{Candidates: &nvidiacomv1beta2.CandidateProgress{Published: 1}} + run.Status.CandidateRefs = []corev1.LocalObjectReference{{Name: "candidate-a"}} + run.Status.LastProgressTime = &progressTime + if err := f.client.Status().Update(t.Context(), run); err != nil { + t.Fatalf("publish progress status: %v", err) + } + + start := metav1.NewTime(progressTime.Add(time.Minute)) + job.Status.StartTime = &start + job.Status.Active = 1 + if err := f.client.Status().Update(t.Context(), job); err != nil { + t.Fatalf("mark Job running: %v", err) + } + f.reconcile() + run = f.run() + if run.Status.Phase != nvidiacomv1beta2.DGDRRunPhaseRunning || run.Status.StartTime == nil || !run.Status.StartTime.Equal(&start) { + t.Fatalf("running status = %+v", run.Status) + } + if run.Status.Progress == nil || len(run.Status.CandidateRefs) != 1 || run.Status.LastProgressTime == nil { + t.Fatalf("publisher-owned status was lost: %+v", run.Status) + } + + completed := metav1.NewTime(start.Add(5 * time.Minute)) + job = f.job() + job.Status.Active = 0 + job.Status.CompletionTime = &completed + job.Status.Conditions = []batchv1.JobCondition{{ + Type: batchv1.JobComplete, Status: corev1.ConditionTrue, LastTransitionTime: completed, + }} + if err := f.client.Status().Update(t.Context(), job); err != nil { + t.Fatalf("complete Job: %v", err) + } + f.reconcile() + run = f.run() + condition := meta.FindStatusCondition(run.Status.Conditions, nvidiacomv1beta2.DGDRRunConditionCompleted) + if run.Status.Phase != nvidiacomv1beta2.DGDRRunPhaseSucceeded || condition == nil || condition.Status != metav1.ConditionTrue { + t.Fatalf("completed status = %+v", run.Status) + } + if run.Status.CompletionTime == nil || !run.Status.CompletionTime.Equal(&completed) { + t.Fatalf("completionTime = %v, want %v", run.Status.CompletionTime, completed) + } +} + +func TestV1beta2RunControllerClassifiesPublisherProtocolFailure(t *testing.T) { + f := newDGDRRunFixture(t, &fixedRunHardware{info: gpu.GPUInfo{GPUsPerNode: 8}}, dgdrRunForTest(t, map[string]any{ + "search_space": map[string]any{"hardware_sku": "h200_sxm", "gpu_budget": 8}, + })) + f.reconcile() + job := f.job() + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: job.Name + "-pod", + Namespace: job.Namespace, + Labels: map[string]string{batchv1.JobNameLabel: job.Name}, + OwnerReferences: []metav1.OwnerReference{{ + APIVersion: batchv1.SchemeGroupVersion.String(), Kind: "Job", Name: job.Name, UID: job.UID, + Controller: ptr.To(true), + }}, + }, + } + if err := f.client.Create(t.Context(), pod); err != nil { + t.Fatalf("create Job pod: %v", err) + } + pod.Status.ContainerStatuses = []corev1.ContainerStatus{{ + Name: dgdrRunPublisherContainer, + State: corev1.ContainerState{Terminated: &corev1.ContainerStateTerminated{ + ExitCode: int32(dgdrrunpublisher.ExitProtocolViolation), Message: "terminal snapshot missing", + }}, + }} + if err := f.client.Status().Update(t.Context(), pod); err != nil { + t.Fatalf("update Job pod status: %v", err) + } + + failed := metav1.Now() + job.Status.Conditions = []batchv1.JobCondition{{ + Type: batchv1.JobFailed, Status: corev1.ConditionTrue, LastTransitionTime: failed, + }} + if err := f.client.Status().Update(t.Context(), job); err != nil { + t.Fatalf("fail Job: %v", err) + } + f.reconcile() + + run := f.run() + condition := meta.FindStatusCondition(run.Status.Conditions, nvidiacomv1beta2.DGDRRunConditionCompleted) + if run.Status.Phase != nvidiacomv1beta2.DGDRRunPhaseFailed || condition == nil || condition.Reason != dgdrRunReasonProtocolViolation { + t.Fatalf("failed status = %+v", run.Status) + } + if !strings.Contains(condition.Message, "terminal snapshot missing") { + t.Fatalf("failure message = %q, want publisher termination message", condition.Message) + } +} + +func TestDGDRRunResourceNameIsStableAndBounded(t *testing.T) { + short := dgdrRunResourceName("search-1") + if short != "search-1-aisimulate" { + t.Fatalf("short name = %q", short) + } + long := strings.Repeat("a", 200) + one, two := dgdrRunResourceName(long), dgdrRunResourceName(long) + if one != two || len(one) > dgdrRunMaxJobNameLength || !strings.HasSuffix(one, "-aisimulate") { + t.Fatalf("bounded name = %q", one) + } +} + +func containerNamed(t *testing.T, containers []corev1.Container, name string) corev1.Container { + t.Helper() + for _, container := range containers { + if container.Name == name { + return container + } + } + t.Fatalf("container %q not found", name) + return corev1.Container{} +} + +func hasMount(mounts []corev1.VolumeMount, name string) bool { + for _, mount := range mounts { + if mount.Name == name { + return true + } + } + return false +} + +func hasMountAt(mounts []corev1.VolumeMount, name, path string) bool { + for _, mount := range mounts { + if mount.Name == name && mount.MountPath == path { + return true + } + } + return false +} diff --git a/deploy/operator/internal/controller/setup.go b/deploy/operator/internal/controller/setup.go index a8704ddbca64..6fd8595c2f6e 100644 --- a/deploy/operator/internal/controller/setup.go +++ b/deploy/operator/internal/controller/setup.go @@ -11,6 +11,7 @@ import ( configv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/config/v1alpha1" lpxcontroller "github.com/ai-dynamo/dynamo/deploy/operator/internal/controller/lpx" commoncontroller "github.com/ai-dynamo/dynamo/deploy/operator/internal/controller_common" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/features" "github.com/ai-dynamo/dynamo/deploy/operator/internal/gpu" "github.com/ai-dynamo/dynamo/deploy/operator/internal/modelendpoint" "github.com/ai-dynamo/dynamo/deploy/operator/internal/secret" @@ -44,6 +45,15 @@ type DynamoGraphDeploymentRequestSetupOptions struct { OperatorImagePullPolicy corev1.PullPolicy } +type DynamoGraphDeploymentRunSetupOptions struct { + SetupOptions + GPUDiscoveryCache *gpu.GPUDiscoveryCache + GPUDiscovery *gpu.GPUDiscovery + DynamoVersion string + OperatorImage string + OperatorImagePullPolicy corev1.PullPolicy +} + type DynamoModelSetupOptions struct { SetupOptions ModelEndpointClient *modelendpoint.Client @@ -63,6 +73,20 @@ func (o DynamoGraphDeploymentRequestSetupOptions) gpuDiscovery() *gpu.GPUDiscove return gpu.NewGPUDiscovery(gpu.ScrapeMetricsEndpoint) } +func (o DynamoGraphDeploymentRunSetupOptions) gpuDiscoveryCache() *gpu.GPUDiscoveryCache { + if o.GPUDiscoveryCache != nil { + return o.GPUDiscoveryCache + } + return gpu.NewGPUDiscoveryCache() +} + +func (o DynamoGraphDeploymentRunSetupOptions) gpuDiscovery() *gpu.GPUDiscovery { + if o.GPUDiscovery != nil { + return o.GPUDiscovery + } + return gpu.NewGPUDiscovery(gpu.ScrapeMetricsEndpoint) +} + func (o DynamoModelSetupOptions) modelEndpointClient() *modelendpoint.Client { if o.ModelEndpointClient != nil { return o.ModelEndpointClient @@ -133,6 +157,46 @@ func SetupDynamoGraphDeploymentRequest(mgr ctrl.Manager, opts DynamoGraphDeploym return nil } +// SetupDynamoGraphDeploymentRequestV1beta2 registers the controller that turns native v1beta2 Requests +// into Runs. It fails at startup, with a clear message, when the cluster does not serve the v1beta2 APIs. +func SetupDynamoGraphDeploymentRequestV1beta2(mgr ctrl.Manager, opts SetupOptions) error { + if err := requireDGDRV1beta2API(mgr); err != nil { + return err + } + if err := (&DynamoGraphDeploymentRequestV1beta2Reconciler{ + Client: mgr.GetClient(), + Config: opts.Config, + RuntimeConfig: opts.RuntimeConfig, + }).SetupWithManager(mgr); err != nil { + return fmt.Errorf("unable to create DynamoGraphDeploymentRequest v1beta2 controller: %w", err) + } + return nil +} + +// SetupDynamoGraphDeploymentRunV1beta2 registers the controller that executes immutable +// v1beta2 Runs as AISimulate Jobs and observes their lifecycle. +func SetupDynamoGraphDeploymentRunV1beta2(mgr ctrl.Manager, opts DynamoGraphDeploymentRunSetupOptions) error { + if err := (&DynamoGraphDeploymentRunV1beta2Reconciler{ + Client: mgr.GetClient(), + Config: opts.Config, + RuntimeConfig: opts.RuntimeConfig, + Hardware: &gpu.Discoverer{ + Reader: mgr.GetClient(), + DCGM: opts.gpuDiscovery(), + Cache: opts.gpuDiscoveryCache(), + NodeLabels: func() bool { + return opts.RuntimeConfig != nil && opts.RuntimeConfig.Gate.Enabled(features.GPUDiscovery) + }, + }, + DynamoVersion: opts.DynamoVersion, + OperatorImage: opts.OperatorImage, + OperatorImagePullPolicy: opts.OperatorImagePullPolicy, + }).SetupWithManager(mgr); err != nil { + return fmt.Errorf("unable to create DynamoGraphDeploymentRun v1beta2 controller: %w", err) + } + return nil +} + func SetupDynamoModel(mgr ctrl.Manager, opts DynamoModelSetupOptions) error { if err := (&DynamoModelReconciler{ Client: mgr.GetClient(), diff --git a/deploy/operator/internal/controller/suite_envtest_test.go b/deploy/operator/internal/controller/suite_envtest_test.go index 45651e30d0f0..0a6c4633cc74 100644 --- a/deploy/operator/internal/controller/suite_envtest_test.go +++ b/deploy/operator/internal/controller/suite_envtest_test.go @@ -9,10 +9,14 @@ package controller import ( + "context" "os" "testing" + nvidiacomv1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/gpu" "github.com/ai-dynamo/dynamo/deploy/operator/internal/testing/operatorenv" + webhookdefaulting "github.com/ai-dynamo/dynamo/deploy/operator/internal/webhook/defaulting" webhooksetup "github.com/ai-dynamo/dynamo/deploy/operator/internal/webhook/setup" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -24,6 +28,16 @@ import ( const admissionBypassUsername = "operatorenv-controller-admission-bypass" +// The controller specs create native v1beta2 Requests without an image or hardware. Admission +// completes them from the operator's configuration, as it does in production. +const controllerAISimulateImage = "registry.example/aisimulate:1.2.0" + +type fixedHardwareDiscoverer struct{} + +func (fixedHardwareDiscoverer) Discover(context.Context, nvidiacomv1beta1.GPUSKUType) (*gpu.GPUInfo, error) { + return &gpu.GPUInfo{System: "h200_sxm", GPUsPerNode: 8, NodesWithGPUs: 1}, nil +} + var ( sharedEnv = operatorenv.New(operatorenv.Options{ Admission: operatorenv.AdmissionWebhooks{ @@ -44,6 +58,10 @@ func setupProductionWebhooks(mgr ctrl.Manager, opts operatorenv.WebhookSetupOpti RuntimeConfig: opts.RuntimeConfig, OperatorVersion: opts.OperatorVersion, OperatorPrincipal: opts.OperatorPrincipal, + DGDRV1beta2: webhookdefaulting.DGDRV1beta2Options{ + AISimulateImage: controllerAISimulateImage, + Hardware: fixedHardwareDiscoverer{}, + }, }) } diff --git a/deploy/operator/internal/dgdcreconcile/diff.go b/deploy/operator/internal/dgdcreconcile/diff.go new file mode 100644 index 000000000000..b9de85216190 --- /dev/null +++ b/deploy/operator/internal/dgdcreconcile/diff.go @@ -0,0 +1,120 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package dgdcreconcile computes the create/delete diff between the candidates a run +// snapshot wants to exist and the DynamoGraphDeploymentCandidates (DGDCs) observed in +// the cluster. +// +// Identity is the stable candidate id carried by the snapshot, which depends only on +// the evaluated point -- never on rank or list position. A DGDC is immutable once its +// initial status is populated, so an identity present on both sides needs no action: +// there is deliberately no "update" action. Reordering candidates is expressed only +// through the ordered candidate references on the run status, which this package does +// not touch. +// +// The package is pure (no Kubernetes types, no I/O) so the diff can be tested on its +// own; callers fetch current state and apply the returned Actions. +package dgdcreconcile + +import "fmt" + +// DiffInputError means the desired or current set was malformed (e.g. a duplicate or +// empty identity). Retrying will not fix it, so callers treat it as terminal. +type DiffInputError struct { + msg string +} + +func (e *DiffInputError) Error() string { return e.msg } + +func newDiffInputError(format string, args ...any) *DiffInputError { + return &DiffInputError{msg: fmt.Sprintf(format, args...)} +} + +type DesiredCandidate struct { + // ID is the stable candidate identity from the snapshot. + ID string + // Spec is the DGD manifest (YAML) to materialize. + Spec string + // Parameters and Metrics are the immutable evaluation facts recorded in the + // DGDC status at creation. + Parameters map[string]any + Metrics map[string]any +} + +// CurrentDGDC is one DGDC observed in the cluster. ID is read back from the label +// recorded at creation, never recomputed. +type CurrentDGDC struct { + Name string + ID string + // Incomplete means the DGDC exists but its initial status was never populated (for + // example the process stopped between creating it and writing its status). Its spec + // is immutable, so only the status is completed. + Incomplete bool +} + +// Actions is what the caller must apply. There are no updates: DGDCs are immutable. +type Actions struct { + // Creates are desired candidates with no DGDC yet. + Creates []DesiredCandidate + // Incomplete are desired candidates whose DGDC exists but whose initial status was + // never populated. The caller finishes the status (never the spec) before the + // candidate is referenced from the run. + Incomplete []DesiredCandidate + Deletes []string // DGDC names +} + +// ComputeActions diffs desired against current. Output order follows the input order +// so results are deterministic. +func ComputeActions(desired []DesiredCandidate, current []CurrentDGDC) (Actions, error) { + desiredIDs := make(map[string]struct{}, len(desired)) + for _, candidate := range desired { + if candidate.ID == "" { + return Actions{}, newDiffInputError("desired candidate has an empty id") + } + if _, dup := desiredIDs[candidate.ID]; dup { + return Actions{}, newDiffInputError("duplicate id in desired set: %s", candidate.ID) + } + desiredIDs[candidate.ID] = struct{}{} + } + currentIDs := make(map[string]struct{}, len(current)) + incomplete := make(map[string]bool, len(current)) + for _, existing := range current { + if existing.ID == "" { + return Actions{}, newDiffInputError("DGDC %s has no candidate id", existing.Name) + } + if _, dup := currentIDs[existing.ID]; dup { + return Actions{}, newDiffInputError("duplicate id in current set: %s", existing.ID) + } + currentIDs[existing.ID] = struct{}{} + incomplete[existing.ID] = existing.Incomplete + } + + var actions Actions + for _, candidate := range desired { + if _, exists := currentIDs[candidate.ID]; !exists { + actions.Creates = append(actions.Creates, candidate) + } else if incomplete[candidate.ID] { + actions.Incomplete = append(actions.Incomplete, candidate) + } + } + for _, existing := range current { + if _, wanted := desiredIDs[existing.ID]; !wanted { + actions.Deletes = append(actions.Deletes, existing.Name) + } + } + return actions, nil +} diff --git a/deploy/operator/internal/dgdcreconcile/diff_test.go b/deploy/operator/internal/dgdcreconcile/diff_test.go new file mode 100644 index 000000000000..3d0f155e2173 --- /dev/null +++ b/deploy/operator/internal/dgdcreconcile/diff_test.go @@ -0,0 +1,123 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dgdcreconcile + +import ( + "errors" + "reflect" + "testing" +) + +func d(ids ...string) []DesiredCandidate { + out := make([]DesiredCandidate, 0, len(ids)) + for _, id := range ids { + out = append(out, DesiredCandidate{ID: id, Spec: "spec-" + id}) + } + return out +} + +func c(pairs ...string) []CurrentDGDC { + var out []CurrentDGDC + for i := 0; i+1 < len(pairs); i += 2 { + out = append(out, CurrentDGDC{Name: pairs[i], ID: pairs[i+1]}) + } + return out +} + +func createIDs(a Actions) []string { + var ids []string + for _, x := range a.Creates { + ids = append(ids, x.ID) + } + return ids +} + +func TestComputeActions(t *testing.T) { + tests := []struct { + name string + desired []DesiredCandidate + current []CurrentDGDC + wantCreates []string + wantIncomplete []string + wantDeletes []string + }{ + {"empty both", nil, nil, nil, nil, nil}, + {"all new", d("a", "b"), nil, []string{"a", "b"}, nil, nil}, + {"all gone", nil, c("n-a", "a", "n-b", "b"), nil, nil, []string{"n-a", "n-b"}}, + {"steady state needs no action", d("a", "b"), c("n-a", "a", "n-b", "b"), nil, nil, nil}, + {"reordering desired needs no action", d("b", "a"), c("n-a", "a", "n-b", "b"), nil, nil, nil}, + {"incomplete DGDC gets its status finished, not a new spec", d("a", "b"), []CurrentDGDC{{Name: "n-a", ID: "a"}, {Name: "n-b", ID: "b", Incomplete: true}}, nil, []string{"b"}, nil}, + {"mixed", d("a", "c"), c("n-a", "a", "n-b", "b"), []string{"c"}, nil, []string{"n-b"}}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + got, err := ComputeActions(tc.desired, tc.current) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !reflect.DeepEqual(createIDs(got), tc.wantCreates) { + t.Errorf("creates = %v, want %v", createIDs(got), tc.wantCreates) + } + var gotIncomplete []string + for _, x := range got.Incomplete { + gotIncomplete = append(gotIncomplete, x.ID) + } + if !reflect.DeepEqual(gotIncomplete, tc.wantIncomplete) { + t.Errorf("incomplete = %v, want %v", gotIncomplete, tc.wantIncomplete) + } + if !reflect.DeepEqual(got.Deletes, tc.wantDeletes) { + t.Errorf("deletes = %v, want %v", got.Deletes, tc.wantDeletes) + } + }) + } +} + +func TestComputeActionsInputErrors(t *testing.T) { + tests := []struct { + name string + desired []DesiredCandidate + current []CurrentDGDC + }{ + {"duplicate desired", d("a", "a"), nil}, + {"empty desired id", d(""), nil}, + {"duplicate current", nil, c("n1", "a", "n2", "a")}, + {"empty current id", nil, c("n1", "")}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + _, err := ComputeActions(tc.desired, tc.current) + var inputErr *DiffInputError + if !errors.As(err, &inputErr) { + t.Fatalf("want *DiffInputError, got %v", err) + } + }) + } +} + +func TestComputeActionsDeterministicOrder(t *testing.T) { + got, err := ComputeActions(d("z", "m", "a"), c("n-q", "q", "n-b", "b")) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(createIDs(got), []string{"z", "m", "a"}) { + t.Errorf("creates order = %v", createIDs(got)) + } + if !reflect.DeepEqual(got.Deletes, []string{"n-q", "n-b"}) { + t.Errorf("deletes order = %v", got.Deletes) + } +} diff --git a/deploy/operator/internal/dgdrrunpublisher/doc.go b/deploy/operator/internal/dgdrrunpublisher/doc.go new file mode 100644 index 000000000000..18b02a637102 --- /dev/null +++ b/deploy/operator/internal/dgdrrunpublisher/doc.go @@ -0,0 +1,62 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package dgdrrunpublisher is the sidecar that turns the Sweeper's pod-local desired +// state snapshot into Kubernetes state: it creates/deletes DynamoGraphDeploymentCandidates +// and updates the ordered candidate references and progress on +// DynamoGraphDeploymentRun.status. +// +// It is a controller-runtime controller for exactly one DynamoGraphDeploymentRun, with +// two loops that meet at one workqueue key (the Run's namespace/name): +// +// 1. The input observer (Observer) checks the snapshot file the Sweeper writes into a +// shared emptyDir. On a semantic change it atomically stores the new desired state +// (InputStore) and enqueues the key. Unchanged input enqueues nothing and backs the +// check interval off; a change resets it. +// 2. The Reconciler consumes the key. Informer events for the Run, its candidates and +// this Pod enqueue the same key. It reads the latest stored input and the cached +// cluster state, then creates missing candidates with server-side apply, completes +// candidate status (its metrics), updates the Run's published-candidate count, progress +// time and ordered references only where they differ, and finally deletes obsolete candidates. Failed writes are +// retried by the workqueue with backoff. +// +// The publisher's own writes come back as watch events and the cache can trail them. The +// Reconciler keeps no record of what it wrote: it decides from the current observations, +// constrains its status writes to the resourceVersion it observed (a stale decision +// conflicts and is retried through the workqueue), and ignores candidate update events; +// see Reconciler. +// +// The manager's cache covers only this Run, its candidates (by the run-UID label) and the +// publisher's own Pod, so there is no API-server polling and no spec comparison: +// candidate specs are immutable once created. +// +// The sidecar is a regular container of the same Job pod as the Sweeper. The Job only +// completes once the publisher has exited, so a successful exit acknowledges that the +// final snapshot was reconciled. The run controller derives the terminal Completed +// condition from the Job result; the publisher never writes a phase. +// +// Termination (see Reconciler): +// +// terminal Succeeded snapshot reconciled -> exit 0 +// terminal Failed snapshot reconciled -> exit ExitRunFailed +// Sweeper exited non-zero, no terminal snapshot -> reconcile last snapshot, exit 0 +// (the Sweeper's exit fails the Job) +// Sweeper exited zero, no terminal snapshot -> reconcile last snapshot, +// exit ExitProtocolViolation +// invalid or regressing snapshot, Sweeper exited -> exit ExitProtocolViolation +// final/last snapshot cannot be reconciled -> exit ExitReconcileFailed +package dgdrrunpublisher diff --git a/deploy/operator/internal/dgdrrunpublisher/input.go b/deploy/operator/internal/dgdrrunpublisher/input.go new file mode 100644 index 000000000000..9d3183638a8a --- /dev/null +++ b/deploy/operator/internal/dgdrrunpublisher/input.go @@ -0,0 +1,194 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dgdrrunpublisher + +import ( + "context" + "errors" + "os" + "reflect" + "sync" + "time" +) + +// Input is the latest accepted desired state plus the most recent problem reading the +// input. Snapshot is never mutated after it is stored, so readers need no copy. +type Input struct { + // Snapshot is the last valid snapshot, or nil if none has been accepted yet. + Snapshot *Snapshot + // Err is why the most recent read failed (invalid or regressing snapshot, or an I/O + // error). It is cleared by the next valid read. + Err error +} + +// InputStore atomically holds the latest desired state for the reconciler. +type InputStore struct { + mu sync.RWMutex + input Input +} + +// Load returns the current input. +func (s *InputStore) Load() Input { + s.mu.RLock() + defer s.mu.RUnlock() + return s.input +} + +// accept stores snap as the latest desired state if it differs semantically from the +// current one, and reports whether it did. A snapshot that moves progress backwards is +// rejected and recorded as the input error; the last valid snapshot stays in place. +func (s *InputStore) accept(snap *Snapshot) (changed bool) { + s.mu.Lock() + defer s.mu.Unlock() + if err := checkProgress(s.input.Snapshot, snap); err != nil { + s.input.Err = err + return false + } + s.input.Err = nil + if s.input.Snapshot != nil && sameContent(s.input.Snapshot, snap) { + return false + } + s.input.Snapshot = snap + return true +} + +// fail records why the input could not be read, keeping the last valid snapshot. +func (s *InputStore) fail(err error) { + s.mu.Lock() + defer s.mu.Unlock() + s.input.Err = err +} + +// sameContent compares two snapshots ignoring the write timestamp: rewriting identical +// content with a new timestamp is not a change the cluster needs to hear about. +func sameContent(a, b *Snapshot) bool { + left, right := *a, *b + left.Timestamp, right.Timestamp = "", "" + return reflect.DeepEqual(left, right) +} + +// checkProgress rejects a snapshot that moves progress backwards relative to the +// previously accepted one: an older round, or fewer evaluated points within the same +// round. A terminal snapshot is exempt from the evaluated check because its count is the +// search's final total, not the live counter. +func checkProgress(previous, next *Snapshot) error { + if previous == nil { + return nil + } + if next.Progress.Round < previous.Progress.Round { + return violation("snapshot round regressed from %d to %d", previous.Progress.Round, next.Progress.Round) + } + if !next.Run.Terminal && next.Progress.Round == previous.Progress.Round && next.Progress.Evaluated < previous.Progress.Evaluated { + return violation("snapshot evaluated count regressed from %d to %d in round %d", previous.Progress.Evaluated, next.Progress.Evaluated, previous.Progress.Round) + } + return nil +} + +// Observer is the input-observation loop: it checks the Sweeper's snapshot file and, on a +// semantic change, stores it and enqueues the run key. It never talks to the API server. +type Observer struct { + Store *InputStore + Path string + // OnError reports a problem reading the snapshot, once per distinct message. It is + // optional; the manager wires it to the controller-runtime logger. + OnError func(err error) + // Enqueue asks the controller to reconcile the run. It must not block. + Enqueue func() + // MinInterval is the check interval after a change; it doubles while the input is + // unchanged, up to MaxInterval. + MinInterval time.Duration + MaxInterval time.Duration + + mu sync.Mutex // serializes Check between the loop and the reconciler + lastErr string +} + +// Check reads the snapshot once and reports whether it was a semantic change (in which +// case it has been stored and the run key enqueued). A missing file is not an error. The +// reconciler also calls it directly once the Sweeper has exited, to be certain the +// Sweeper's final write has been seen. +func (o *Observer) Check() bool { + o.mu.Lock() + defer o.mu.Unlock() + + data, err := os.ReadFile(o.Path) + if errors.Is(err, os.ErrNotExist) { + return false + } + if err != nil { + o.noteError(err) + o.Store.fail(err) + return false + } + snap, err := ParseSnapshot(data) + if err != nil { + o.noteError(err) + o.Store.fail(err) + return false + } + changed := o.Store.accept(snap) + if err := o.Store.Load().Err; err != nil { + o.noteError(err) + return false + } + o.lastErr = "" + if changed && o.Enqueue != nil { + o.Enqueue() + } + return changed +} + +// noteError logs a problem once per distinct message rather than on every check. +func (o *Observer) noteError(err error) { + if err.Error() == o.lastErr { + return + } + o.lastErr = err.Error() + if o.OnError != nil { + o.OnError(err) + } +} + +// NextInterval is the delay before the next check: it resets to min after a change and +// otherwise doubles up to max. +func NextInterval(current, minimum, maximum time.Duration, changed bool) time.Duration { + if changed || current < minimum { + return minimum + } + next := current * 2 + if next > maximum { + return maximum + } + return next +} + +// Run checks the snapshot until ctx is done. +func (o *Observer) Run(ctx context.Context) error { + interval := o.MinInterval + timer := time.NewTimer(0) + defer timer.Stop() + for { + select { + case <-ctx.Done(): + return nil + case <-timer.C: + } + interval = NextInterval(interval, o.MinInterval, o.MaxInterval, o.Check()) + timer.Reset(interval) + } +} diff --git a/deploy/operator/internal/dgdrrunpublisher/input_test.go b/deploy/operator/internal/dgdrrunpublisher/input_test.go new file mode 100644 index 000000000000..8b13734c92fa --- /dev/null +++ b/deploy/operator/internal/dgdrrunpublisher/input_test.go @@ -0,0 +1,179 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dgdrrunpublisher + +import ( + "context" + "errors" + "os" + "path/filepath" + "testing" + "time" +) + +func newObserver(t *testing.T) (*Observer, *InputStore, string, *int) { + t.Helper() + dir := t.TempDir() + enqueued := 0 + store := &InputStore{} + return &Observer{ + Store: store, + Path: filepath.Join(dir, SnapshotFileName), + Enqueue: func() { enqueued++ }, + MinInterval: time.Millisecond, + MaxInterval: 4 * time.Millisecond, + }, store, dir, &enqueued +} + +func TestMissingSnapshotIsNotAChange(t *testing.T) { + o, store, _, enqueued := newObserver(t) + if o.Check() || *enqueued != 0 || store.Load().Snapshot != nil || store.Load().Err != nil { + t.Fatalf("missing file: input=%+v enqueued=%d", store.Load(), *enqueued) + } +} + +func TestChangedSnapshotIsStoredAndEnqueuedOnce(t *testing.T) { + o, store, dir, enqueued := newObserver(t) + writeSnapshot(t, dir, snapshotJSON(t, PhaseRunning, cand{id: "a"})) + if !o.Check() || *enqueued != 1 { + t.Fatalf("first snapshot: enqueued=%d", *enqueued) + } + if store.Load().Snapshot == nil || store.Load().Snapshot.Candidates[0].ID != "a" { + t.Fatalf("snapshot not stored: %+v", store.Load()) + } + if o.Check() || *enqueued != 1 { + t.Fatalf("unchanged input must not enqueue again: enqueued=%d", *enqueued) + } + writeSnapshot(t, dir, snapshotJSON(t, PhaseRunning, cand{id: "a"}, cand{id: "b"})) + if !o.Check() || *enqueued != 2 { + t.Fatalf("changed input must enqueue: enqueued=%d", *enqueued) + } +} + +func TestRewriteWithOnlyANewTimestampIsNotAChange(t *testing.T) { + o, _, dir, enqueued := newObserver(t) + writeSnapshot(t, dir, withTimestamp(t, snapshotJSON(t, PhaseRunning, cand{id: "a"}), "2026-10-05T21:35:12Z")) + o.Check() + writeSnapshot(t, dir, withTimestamp(t, snapshotJSON(t, PhaseRunning, cand{id: "a"}), "2026-10-05T21:35:14Z")) + if o.Check() || *enqueued != 1 { + t.Fatalf("a new timestamp alone is not a semantic change: enqueued=%d", *enqueued) + } +} + +func TestInvalidSnapshotKeepsTheLastValidOneAndIsRecorded(t *testing.T) { + o, store, dir, enqueued := newObserver(t) + writeSnapshot(t, dir, snapshotJSON(t, PhaseRunning, cand{id: "a"})) + o.Check() + writeSnapshot(t, dir, []byte(`{"schemaVersion":2}`)) + if o.Check() || *enqueued != 1 { + t.Fatalf("an invalid snapshot must not enqueue: enqueued=%d", *enqueued) + } + in := store.Load() + if !errors.Is(in.Err, ErrProtocolViolation) || in.Snapshot == nil || in.Snapshot.Candidates[0].ID != "a" { + t.Fatalf("input = %+v", in) + } + writeSnapshot(t, dir, snapshotJSON(t, PhaseRunning, cand{id: "a"}, cand{id: "b"})) + if !o.Check() || store.Load().Err != nil { + t.Fatalf("a valid snapshot must clear the error: %+v", store.Load()) + } +} + +func TestRegressingSnapshotIsRejected(t *testing.T) { + for name, regressed := range map[string][2]int{"older round": {2, 99}, "fewer evaluated in the same round": {3, 4}} { + t.Run(name, func(t *testing.T) { + o, store, dir, enqueued := newObserver(t) + writeProgress(t, dir, PhaseRunning, 3, 5) + o.Check() + writeProgress(t, dir, PhaseRunning, regressed[0], regressed[1]) + if o.Check() || *enqueued != 1 { + t.Fatalf("a regressing snapshot must not be stored: enqueued=%d", *enqueued) + } + in := store.Load() + if !errors.Is(in.Err, ErrProtocolViolation) || ExitCode(in.Err) != ExitProtocolViolation { + t.Fatalf("err = %v", in.Err) + } + if in.Snapshot.Progress.Round != 3 || in.Snapshot.Progress.Evaluated != 5 { + t.Fatalf("the last valid snapshot must stay: %+v", in.Snapshot.Progress) + } + }) + } +} + +func TestTerminalSnapshotMayReportItsFinalEvaluatedTotal(t *testing.T) { + o, store, dir, _ := newObserver(t) + writeProgress(t, dir, PhaseRunning, 3, 9) + o.Check() + writeProgress(t, dir, PhaseSucceeded, 3, 7) + if !o.Check() || store.Load().Err != nil || !store.Load().Snapshot.Run.Terminal { + t.Fatalf("terminal snapshot rejected: %+v", store.Load()) + } +} + +func TestUnreadableSnapshotIsRecordedAsANonViolation(t *testing.T) { + o, store, dir, _ := newObserver(t) + if err := os.Mkdir(filepath.Join(dir, SnapshotFileName), 0o700); err != nil { // reading a directory fails + t.Fatal(err) + } + o.Check() + if err := store.Load().Err; err == nil || errors.Is(err, ErrProtocolViolation) { + t.Fatalf("err = %v, want a plain read error", err) + } +} + +func TestNextIntervalBacksOffWhileUnchangedAndResetsOnChange(t *testing.T) { + minimum, maximum := time.Second, 5*time.Second + got := []time.Duration{} + interval := minimum + for i := 0; i < 5; i++ { + interval = NextInterval(interval, minimum, maximum, false) + got = append(got, interval) + } + want := []time.Duration{2 * time.Second, 4 * time.Second, 5 * time.Second, 5 * time.Second, 5 * time.Second} + for i := range want { + if got[i] != want[i] { + t.Fatalf("intervals = %v, want %v", got, want) + } + } + if NextInterval(5*time.Second, minimum, maximum, true) != minimum { + t.Fatal("a change must reset the interval") + } +} + +func TestObserverRunEnqueuesAChangeAndStopsWithItsContext(t *testing.T) { + o, _, dir, _ := newObserver(t) + events := make(chan struct{}, 4) + o.Enqueue = func() { events <- struct{}{} } + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { done <- o.Run(ctx) }() + writeSnapshot(t, dir, snapshotJSON(t, PhaseRunning, cand{id: "a"})) + select { + case <-events: + case <-time.After(5 * time.Second): + t.Fatal("the observer never reported the snapshot") + } + cancel() + select { + case err := <-done: + if err != nil { + t.Fatalf("Run returned %v", err) + } + case <-time.After(5 * time.Second): + t.Fatal("Run did not stop") + } +} diff --git a/deploy/operator/internal/dgdrrunpublisher/manager.go b/deploy/operator/internal/dgdrrunpublisher/manager.go new file mode 100644 index 000000000000..8c67862a1684 --- /dev/null +++ b/deploy/operator/internal/dgdrrunpublisher/manager.go @@ -0,0 +1,229 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dgdrrunpublisher + +import ( + "context" + "errors" + "fmt" + "path/filepath" + "time" + + v1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" + v1beta2 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta2" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/builder" + "sigs.k8s.io/controller-runtime/pkg/cache" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/manager" + metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" + "sigs.k8s.io/controller-runtime/pkg/predicate" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/controller-runtime/pkg/source" +) + +// Config is everything one publisher needs to know about its run and pod. +type Config struct { + Namespace string + RunName string + // RunUID is the UID of the run this pod serves. The run controller knows it when it + // builds the Job, which is what lets the cache select this run's candidates by label. + RunUID string + PodName string + SweeperContainer string + SnapshotDir string + // MinPollInterval and MaxPollInterval bound the snapshot check interval, which backs + // off from the minimum to the maximum while the input is unchanged. + MinPollInterval time.Duration + MaxPollInterval time.Duration +} + +func (c Config) validate() error { + switch { + case c.Namespace == "", c.RunName == "", c.RunUID == "", c.PodName == "", c.SnapshotDir == "": + return errors.New("namespace, run name, run uid, pod name and snapshot dir are required") + case c.MinPollInterval <= 0 || c.MaxPollInterval < c.MinPollInterval: + return errors.New("poll intervals must be positive and the maximum at least the minimum") + } + return nil +} + +// NewScheme registers every type the publisher reads or writes. +func NewScheme() (*runtime.Scheme, error) { + scheme := runtime.NewScheme() + for _, add := range []func(*runtime.Scheme) error{clientgoscheme.AddToScheme, v1beta1.AddToScheme, v1beta2.AddToScheme} { + if err := add(scheme); err != nil { + return nil, err + } + } + return scheme, nil +} + +// cacheOptions limits the manager's informers to this run, its candidates and its own +// pod: one namespace, plus a field selector on the names of the run and the pod and a +// label selector on the run UID for the candidates. +func cacheOptions(c Config) cache.Options { + return cache.Options{ + DefaultNamespaces: map[string]cache.Config{c.Namespace: {}}, + ByObject: map[client.Object]cache.ByObject{ + &v1beta2.DynamoGraphDeploymentRun{}: { + Field: fields.OneTermEqualSelector("metadata.name", c.RunName), + }, + &v1beta2.DynamoGraphDeploymentCandidate{}: { + Label: labels.SelectorFromSet(labels.Set{LabelRunUID: c.RunUID}), + }, + &corev1.Pod{}: { + Field: fields.OneTermEqualSelector("metadata.name", c.PodName), + }, + }, + } +} + +// ignoreCandidateUpdates drops candidate update events. The publisher is the only writer +// of a candidate: its spec is immutable and its status is written once, so an update is +// the echo of our own write and carries nothing to act on. Creates (including the +// initial list) and deletes still enqueue the run key, so a deleted candidate is +// repaired. +var ignoreCandidateUpdates = predicate.Funcs{ + UpdateFunc: func(event.UpdateEvent) bool { return false }, +} + +// Run starts the controller manager and returns the publisher's outcome (see the package +// comment for its meaning and ExitCode for the exit code), or ctx's error if the process +// is told to stop first. +func Run(ctx context.Context, restConfig *rest.Config, cfg Config) error { + if err := cfg.validate(); err != nil { + return err + } + scheme, err := NewScheme() + if err != nil { + return fmt.Errorf("building scheme: %w", err) + } + shutdownTimeout := 5 * time.Second + mgr, err := ctrl.NewManager(restConfig, ctrl.Options{ + Scheme: scheme, + Cache: cacheOptions(cfg), + Metrics: metricsserver.Options{BindAddress: "0"}, + LeaderElection: false, + GracefulShutdownTimeout: &shutdownTimeout, + }) + if err != nil { + return fmt.Errorf("creating manager: %w", err) + } + + key := types.NamespacedName{Namespace: cfg.Namespace, Name: cfg.RunName} + events := make(chan event.GenericEvent, 1) + store := &InputStore{} + observer := &Observer{ + Store: store, + Path: SnapshotPath(cfg.SnapshotDir), + MinInterval: cfg.MinPollInterval, + MaxInterval: cfg.MaxPollInterval, + OnError: func(err error) { + ctrl.Log.WithName("input-observer").Error(err, "reading snapshot") + }, + Enqueue: func() { + // One pending event is enough: the workqueue deduplicates by key anyway. + select { + case events <- event.GenericEvent{Object: runKeyObject(key)}: + default: + } + }, + } + completion := NewCompletion() + reconciler := &Reconciler{ + Client: mgr.GetClient(), + Writer: &clientWriter{ + Client: mgr.GetClient(), + Reader: mgr.GetAPIReader(), + Namespace: cfg.Namespace, + RunName: cfg.RunName, + RunUID: cfg.RunUID, + }, + Namespace: cfg.Namespace, + RunName: cfg.RunName, + RunUID: cfg.RunUID, + PodName: cfg.PodName, + SweeperContainer: cfg.SweeperContainer, + Observer: observer, + Store: store, + Completion: completion, + } + + podToRun := handler.EnqueueRequestsFromMapFunc(func(context.Context, client.Object) []reconcile.Request { + return []reconcile.Request{{NamespacedName: key}} + }) + if err := ctrl.NewControllerManagedBy(mgr). + Named("dgdr-run-publisher"). + For(&v1beta2.DynamoGraphDeploymentRun{}). + Owns(&v1beta2.DynamoGraphDeploymentCandidate{}, builder.WithPredicates(ignoreCandidateUpdates)). + Watches(&corev1.Pod{}, podToRun). + WatchesRawSource(source.Channel(events, &handler.EnqueueRequestForObject{})). + WithOptions(controller.Options{MaxConcurrentReconciles: 1}). + Complete(reconciler); err != nil { + return fmt.Errorf("building controller: %w", err) + } + if err := mgr.Add(manager.RunnableFunc(observer.Run)); err != nil { + return fmt.Errorf("adding input observer: %w", err) + } + + runCtx, cancel := context.WithCancel(ctx) + defer cancel() + go func() { + select { + case <-completion.Done(): + cancel() // the outcome is known: stop the manager and exit + case <-runCtx.Done(): + } + }() + startErr := mgr.Start(runCtx) + + if completion.Finished() { + return completion.Err() + } + if ctx.Err() != nil { + return ctx.Err() + } + if startErr != nil { + return fmt.Errorf("manager stopped: %w", startErr) + } + return errors.New("manager stopped before the run finished") +} + +// SnapshotPath is where the Sweeper writes the snapshot inside the shared directory. +func SnapshotPath(dir string) string { + return filepath.Join(dir, SnapshotFileName) +} + +// runKeyObject is the object carried by enqueue events; only its name and namespace +// matter to the handler. +func runKeyObject(key types.NamespacedName) client.Object { + run := &v1beta2.DynamoGraphDeploymentRun{} + run.Namespace, run.Name = key.Namespace, key.Name + return run +} diff --git a/deploy/operator/internal/dgdrrunpublisher/outcome.go b/deploy/operator/internal/dgdrrunpublisher/outcome.go new file mode 100644 index 000000000000..aea5b707ebee --- /dev/null +++ b/deploy/operator/internal/dgdrrunpublisher/outcome.go @@ -0,0 +1,114 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dgdrrunpublisher + +import ( + "errors" + "fmt" + "sync" +) + +// Process exit codes of the publisher binary. +const ( + ExitAcknowledged = 0 + ExitReconcileFailed = 1 + ExitProtocolViolation = 3 + ExitRunFailed = 4 +) + +// ErrMissingTerminalSnapshot means the Sweeper exited successfully without publishing a +// terminal snapshot, which violates the producer protocol. +var ErrMissingTerminalSnapshot = fmt.Errorf("%w: sweeper exited without publishing a terminal snapshot", ErrProtocolViolation) + +// ErrRunFailed means the Sweeper published a terminal Failed snapshot. The snapshot is +// reconciled first; the error keeps a producer that reports failure but exits 0 from +// letting the Job complete successfully. +var ErrRunFailed = errors.New("sweeper reported a failed run") + +// ExitCode maps the result of Run to the process exit code. +func ExitCode(err error) int { + switch { + case err == nil: + return ExitAcknowledged + case errors.Is(err, ErrProtocolViolation): + return ExitProtocolViolation + case errors.Is(err, ErrRunFailed): + return ExitRunFailed + default: + return ExitReconcileFailed + } +} + +// SweeperState is the observed state of the Sweeper container in this pod. +type SweeperState struct { + Exited bool + ExitCode int32 +} + +// terminalResult is the outcome once a terminal snapshot has been reconciled. +func terminalResult(phase string) error { + if phase == PhaseFailed { + return ErrRunFailed + } + return nil +} + +// exitResult is the outcome when the Sweeper exited and the last snapshot it wrote (if +// any) has been reconciled, but that snapshot is not terminal. +func exitResult(state SweeperState) error { + if state.ExitCode != 0 { + // Crash or caught failure without a terminal snapshot: the last-known projection + // is reconciled; the Sweeper's own exit code fails the Job. + return nil + } + return ErrMissingTerminalSnapshot +} + +// Completion records the publisher's outcome exactly once and wakes whoever waits on it. +type Completion struct { + once sync.Once + done chan struct{} + err error +} + +// NewCompletion returns an unfinished Completion. +func NewCompletion() *Completion { return &Completion{done: make(chan struct{})} } + +// Finish records err as the outcome. Only the first call has an effect. +func (c *Completion) Finish(err error) { + c.once.Do(func() { + c.err = err + close(c.done) + }) +} + +// Done is closed once the outcome is known. +func (c *Completion) Done() <-chan struct{} { return c.done } + +// Err is the outcome. It is only meaningful after Done is closed. +func (c *Completion) Err() error { return c.err } + +// Finished reports whether the outcome is known. +func (c *Completion) Finished() bool { + select { + case <-c.done: + return true + default: + return false + } +} diff --git a/deploy/operator/internal/dgdrrunpublisher/outcome_test.go b/deploy/operator/internal/dgdrrunpublisher/outcome_test.go new file mode 100644 index 000000000000..7aa0d89a4468 --- /dev/null +++ b/deploy/operator/internal/dgdrrunpublisher/outcome_test.go @@ -0,0 +1,78 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dgdrrunpublisher + +import ( + "errors" + "fmt" + "testing" +) + +func TestExitCodes(t *testing.T) { + cases := []struct { + err error + want int + }{ + {nil, ExitAcknowledged}, + {ErrRunFailed, ExitRunFailed}, + {ErrMissingTerminalSnapshot, ExitProtocolViolation}, + {violation("bad"), ExitProtocolViolation}, + {fmt.Errorf("reconciling last snapshot: %w", violation("bad")), ExitProtocolViolation}, + {errors.New("api down"), ExitReconcileFailed}, + } + for _, tc := range cases { + if got := ExitCode(tc.err); got != tc.want { + t.Errorf("ExitCode(%v) = %d, want %d", tc.err, got, tc.want) + } + } +} + +func TestTerminalResult(t *testing.T) { + if err := terminalResult(PhaseSucceeded); err != nil { + t.Fatalf("succeeded: %v", err) + } + if err := terminalResult(PhaseFailed); !errors.Is(err, ErrRunFailed) { + t.Fatalf("failed: %v", err) + } +} + +func TestExitResultWithoutATerminalSnapshot(t *testing.T) { + if err := exitResult(SweeperState{Exited: true, ExitCode: 137}); err != nil { + t.Fatalf("a crash is reported by the Sweeper's own exit code, not the publisher: %v", err) + } + if err := exitResult(SweeperState{Exited: true}); !errors.Is(err, ErrMissingTerminalSnapshot) { + t.Fatalf("a clean exit without a terminal snapshot is a protocol violation: %v", err) + } +} + +func TestCompletionKeepsTheFirstOutcome(t *testing.T) { + c := NewCompletion() + if c.Finished() { + t.Fatal("finished before Finish") + } + c.Finish(ErrRunFailed) + c.Finish(nil) + select { + case <-c.Done(): + default: + t.Fatal("Done not closed") + } + if !c.Finished() || !errors.Is(c.Err(), ErrRunFailed) { + t.Fatalf("err = %v", c.Err()) + } +} diff --git a/deploy/operator/internal/dgdrrunpublisher/plan.go b/deploy/operator/internal/dgdrrunpublisher/plan.go new file mode 100644 index 000000000000..3f1efa3601f4 --- /dev/null +++ b/deploy/operator/internal/dgdrrunpublisher/plan.go @@ -0,0 +1,155 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dgdrrunpublisher + +import ( + "crypto/sha256" + "encoding/hex" + "slices" + "strings" + "time" + + "github.com/ai-dynamo/dynamo/deploy/operator/internal/dgdcreconcile" +) + +// SnapshotFileName is the file the Sweeper container writes. +const SnapshotFileName = "dgdr_run_snapshot.yaml" + +// maxObjectName is the Kubernetes limit on an object name (DNS subdomain). +const maxObjectName = 253 + +// RunStatus is the part of DynamoGraphDeploymentRun.status the publisher owns: the +// ordered candidate references, the number of published candidates +// (status.progress.candidates.published) and the coarse progress time. CandidateNames is +// in rank order (best first for scalar searches). Published always equals +// len(CandidateNames) in a desired status; it is kept separately so a status whose +// progress block was removed or edited by someone else is noticed and repaired. +// LastProgress is the zero time when the snapshot carries no usable timestamp, in which +// case the existing value is left untouched. +// +// The snapshot's round and evaluated counters have no counterpart in the Run status: it +// reports per-branch rounds and outcome counters that the snapshot does not carry yet, +// so they are not published rather than guessed. +type RunStatus struct { + Published int32 + LastProgress time.Time + CandidateNames []string +} + +// DesiredState is what one snapshot says should exist in the cluster. +type DesiredState struct { + // Candidates are the materialized candidates, in snapshot (rank) order. + Candidates []dgdcreconcile.DesiredCandidate + // Names maps a candidate id to its DGDC name. + Names map[string]string + Status RunStatus + Terminal bool + Phase string + // Failed counts the candidates whose materialization failed. They are not created + // and the Run status has no field for them, so they are only logged. + Failed int +} + +// CandidateName is the DGDC name for a candidate id. It depends only on the run +// incarnation (name and UID) and the stable id, never on rank. The UID component keeps a +// recreated run of the same name from colliding with candidates of the previous one that +// are still waiting for garbage collection. A run name too long to leave room for the +// rest is truncated and a hash of the full run name is added, so names stay unique and +// within the limit. +func CandidateName(runName, runUID, id string) string { + uidSum := sha256.Sum256([]byte(runUID)) + suffix := "-" + hex.EncodeToString(uidSum[:4]) + "-" + id + if len(runName)+len(suffix) <= maxObjectName { + return runName + suffix + } + nameSum := sha256.Sum256([]byte(runName)) + hash := hex.EncodeToString(nameSum[:6]) + room := maxObjectName - len(suffix) - len(hash) - 1 + return strings.TrimRight(runName[:room], "-.") + "-" + hash + suffix +} + +// Desire projects a snapshot onto the desired cluster state of one run incarnation. +// Candidates whose materialization failed are not created; they are only counted in +// DesiredState.Failed. +func Desire(runName, runUID string, snap *Snapshot) DesiredState { + state := DesiredState{ + Names: make(map[string]string, len(snap.Candidates)), + Terminal: snap.Run.Terminal, + Phase: snap.Run.Phase, + } + failed := 0 + var names []string + for _, candidate := range snap.Candidates { + if candidate.Outcome != OutcomeMaterialized { + failed++ + continue + } + name := CandidateName(runName, runUID, candidate.ID) + state.Names[candidate.ID] = name + names = append(names, name) + state.Candidates = append(state.Candidates, dgdcreconcile.DesiredCandidate{ + ID: candidate.ID, + Spec: candidate.Manifest, + Parameters: candidate.Parameters, + Metrics: candidate.Metrics, + }) + } + state.Failed = failed + state.Status = RunStatus{ + Published: int32(len(names)), + LastProgress: progressTime(snap), + CandidateNames: names, + } + return state +} + +// progressTime is the snapshot's own timestamp (written by the Sweeper container in the +// same pod) truncated to the second the API stores, or the zero time when it is missing +// or malformed. Using the snapshot's time instead of the clock keeps reconciliation +// deterministic. +func progressTime(snap *Snapshot) time.Time { + parsed, err := time.Parse(time.RFC3339, snap.Timestamp) + if err != nil { + return time.Time{} + } + return parsed.UTC().Truncate(time.Second) +} + +// WithPreservedProgress returns the status to write. When the snapshot carries no usable +// timestamp, the progress time already known (observed or written earlier) is kept: +// omitting a field from a server-side-apply payload removes it once this manager owns +// it, so "leave it alone" has to be spelled out as "write the same value again". +func (desired RunStatus) WithPreservedProgress(known time.Time) RunStatus { + if desired.LastProgress.IsZero() { + desired.LastProgress = known + } + return desired +} + +// NeedsUpdate reports whether the observed run status differs from the desired one, so +// the status is only written when something changed. A zero desired LastProgress means +// "leave the observed value alone". +func (desired RunStatus) NeedsUpdate(observed RunStatus) bool { + if desired.Published != observed.Published { + return true + } + if !desired.LastProgress.IsZero() && !desired.LastProgress.Equal(observed.LastProgress) { + return true + } + return !slices.Equal(desired.CandidateNames, observed.CandidateNames) +} diff --git a/deploy/operator/internal/dgdrrunpublisher/plan_test.go b/deploy/operator/internal/dgdrrunpublisher/plan_test.go new file mode 100644 index 000000000000..c45ba1c6ea78 --- /dev/null +++ b/deploy/operator/internal/dgdrrunpublisher/plan_test.go @@ -0,0 +1,138 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dgdrrunpublisher + +import ( + "reflect" + "strings" + "testing" + "time" +) + +func TestCandidateNameIsBoundedAndUniquePerRunIncarnation(t *testing.T) { + id := "evaluated-point-0123456789ab" + if cn("a") != CandidateName("run", testRunUID, "a") || cn("a") == CandidateName("run", "uid-2", "a") { + t.Fatalf("a recreated run must get different names: %q", cn("a")) + } + long := strings.Repeat("r", 253) + other := strings.Repeat("r", 252) + "s" + name := CandidateName(long, testRunUID, id) + if len(name) > 253 || !strings.HasSuffix(name, "-"+id) || name == CandidateName(other, testRunUID, id) || name != CandidateName(long, testRunUID, id) { + t.Fatalf("name = %q (%d)", name, len(name)) + } +} + +func desire(t *testing.T, data []byte) DesiredState { + t.Helper() + return Desire("run", testRunUID, mustParse(t, data)) +} + +func TestSwapKeepsCandidateNamesAndOnlyReordersReferences(t *testing.T) { + first := desire(t, snapshotJSON(t, PhaseRunning, cand{id: "a"}, cand{id: "b"})) + second := desire(t, snapshotJSON(t, PhaseRunning, cand{id: "b"}, cand{id: "a"})) + if !reflect.DeepEqual(first.Names, second.Names) { + t.Fatalf("names depend on rank: %v vs %v", first.Names, second.Names) + } + if want := []string{cn("b"), cn("a")}; !reflect.DeepEqual(second.Status.CandidateNames, want) { + t.Fatalf("ordered references = %v, want %v", second.Status.CandidateNames, want) + } + if !second.Status.NeedsUpdate(first.Status) { + t.Fatal("a reorder must update the run's references") + } +} + +func TestFailedMaterializationIsNotDesiredButCounted(t *testing.T) { + state := desire(t, snapshotJSON(t, PhaseRunning, cand{id: "a"}, cand{id: "x", failed: true})) + if len(state.Candidates) != 1 || state.Candidates[0].ID != "a" { + t.Fatalf("candidates = %+v", state.Candidates) + } + if !reflect.DeepEqual(state.Status.CandidateNames, []string{cn("a")}) { + t.Fatalf("refs must not include the failed candidate: %v", state.Status.CandidateNames) + } + if state.Failed != 1 { + t.Fatalf("failed = %d, want 1", state.Failed) + } + if state.Status.Published != 1 { + t.Fatalf("published = %d, want 1: failed candidates are not published", state.Status.Published) + } +} + +func TestPublishedCountsTheReferencedCandidates(t *testing.T) { + state := desire(t, snapshotJSON(t, PhaseRunning, cand{id: "a"}, cand{id: "b"})) + if got, want := state.Status.Published, int32(len(state.Status.CandidateNames)); got != want || got != 2 { + t.Fatalf("published = %d, refs = %d, want 2", got, want) + } +} + +func TestLastProgressComesFromTheSnapshotTimestamp(t *testing.T) { + data := withTimestamp(t, snapshotJSON(t, PhaseRunning, cand{id: "a"}), "2026-10-05T21:35:12Z") + want := time.Date(2026, 10, 5, 21, 35, 12, 0, time.UTC) + if got := desire(t, data).Status.LastProgress; !got.Equal(want) { + t.Fatalf("LastProgress = %v, want %v", got, want) + } +} + +func TestMissingOrMalformedTimestampLeavesProgressTimeAlone(t *testing.T) { + for _, timestamp := range []string{"", "yesterday"} { + state := desire(t, withTimestamp(t, snapshotJSON(t, PhaseRunning, cand{id: "a"}), timestamp)) + if !state.Status.LastProgress.IsZero() { + t.Fatalf("timestamp %q: LastProgress = %v, want zero", timestamp, state.Status.LastProgress) + } + observed := state.Status + observed.LastProgress = time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + if state.Status.NeedsUpdate(observed) { + t.Fatalf("timestamp %q: an unknown progress time must not force an update", timestamp) + } + } +} + +func TestNeedsUpdateOnlyWhenSomethingDiffers(t *testing.T) { + base := desire(t, withTimestamp(t, snapshotJSON(t, PhaseRunning, cand{id: "a"}), "2026-10-05T21:35:12Z")).Status + if base.NeedsUpdate(base) { + t.Fatal("identical status needs no write") + } + changes := map[string]func(*RunStatus){ + "published": func(s *RunStatus) { s.Published++ }, + "progress time": func(s *RunStatus) { s.LastProgress = s.LastProgress.Add(time.Second) }, + "references": func(s *RunStatus) { s.CandidateNames = nil }, + } + for name, mutate := range changes { + observed := base + observed.CandidateNames = append([]string(nil), base.CandidateNames...) + mutate(&observed) + if !base.NeedsUpdate(observed) { + t.Errorf("%s: a difference must trigger an update", name) + } + } +} + +func TestWithPreservedProgressKeepsTheKnownTimeOnlyWhenTheSnapshotHasNone(t *testing.T) { + known := time.Date(2026, 10, 5, 21, 35, 12, 0, time.UTC) + without := desire(t, withTimestamp(t, snapshotJSON(t, PhaseRunning, cand{id: "a"}), "")).Status + if got := without.WithPreservedProgress(known).LastProgress; !got.Equal(known) { + t.Fatalf("a snapshot without a timestamp must keep the known time, got %v", got) + } + newer := time.Date(2026, 10, 5, 21, 40, 0, 0, time.UTC) + with := desire(t, withTimestamp(t, snapshotJSON(t, PhaseRunning, cand{id: "a"}), "2026-10-05T21:40:00Z")).Status + if got := with.WithPreservedProgress(known).LastProgress; !got.Equal(newer) { + t.Fatalf("a snapshot with a timestamp must win, got %v", got) + } + if got := without.WithPreservedProgress(time.Time{}).LastProgress; !got.IsZero() { + t.Fatalf("nothing known, nothing to preserve, got %v", got) + } +} diff --git a/deploy/operator/internal/dgdrrunpublisher/reconciler.go b/deploy/operator/internal/dgdrrunpublisher/reconciler.go new file mode 100644 index 000000000000..85cd6ad4efa8 --- /dev/null +++ b/deploy/operator/internal/dgdrrunpublisher/reconciler.go @@ -0,0 +1,341 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dgdrrunpublisher + +import ( + "context" + "errors" + "fmt" + "strings" + + dgdv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1" + v1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" + v1beta2 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta2" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/dgdcreconcile" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +const ( + // LabelRunUID ties a candidate to one incarnation of its run. The run name can be + // reused after a delete and recreate; the UID cannot, so candidates left over from a + // previous run of the same name are never mistaken for current ones. The manager's + // cache selects candidates by this label. + LabelRunUID = "nvidia.com/dgdr-run-uid" + // LabelCandidateID records the stable candidate id (the identity read back on list). + LabelCandidateID = "nvidia.com/dgdr-candidate-id" + + // AnnotationAdditionalResources carries the companion resources (for example + // generated ConfigMaps) rendered next to the DGD, as multi-document YAML. It is the + // same key and format the DGDR controller uses. + AnnotationAdditionalResources = "dgdr.nvidia.com/additional-resources" + + // fieldOwner is the server-side-apply field manager of the publisher. + fieldOwner = "dgdr-run-publisher" + + // finalReconcileAttempts bounds the retries once the Sweeper has exited: nothing will + // rewrite the snapshot, so a persistent failure must end the process. + finalReconcileAttempts = 3 +) + +// Reconciler reconciles the one DynamoGraphDeploymentRun this publisher serves. +// +// Client is the manager's client: reads come from the scoped informer cache (this Run, +// its candidates, this Pod). Writes go through Writer to the API server. It needs RBAC +// (namespaced Role bound to the run's ServiceAccount): runs get/list/watch and +// runs/status patch (server-side apply); candidates get/list/watch/create/patch/delete +// and candidates/status patch; pods get/list/watch (own pod, for Sweeper crash +// detection; the cache narrows list/watch to it with a field selector because generated +// Job pod names cannot be listed in resourceNames). +// +// Write echoes and cache lag: the publisher's own writes come back as watch events, and +// the cache can trail them. The reconciler keeps no record of its writes. Every pass +// decides from the current observations (the cached run and candidates, the latest input +// snapshot), so a repeated pass while the cache catches up may repeat an idempotent +// write, which is harmless, and an echo of a write finds nothing left to do. Anything +// that changes behind the publisher's back (a deleted candidate, a modified run status) +// is repaired on the next pass. +// +// Stale reads: every status write is constrained to the resourceVersion the decision was +// based on (the cached run, the cached candidate, or the version read back from the API +// server for a candidate applied a moment ago). A decision made from an older cache +// observation fails with a conflict instead of overwriting newer state, and is retried +// through the workqueue once the informer has delivered the newer version. A candidate's +// status is only populated when the API server's own copy, read right after the apply, +// has none yet, so a stale cache can never make the publisher rewrite populated status. +type Reconciler struct { + Client client.Client + Writer writer + Namespace string + RunName string + RunUID string + PodName string + SweeperContainer string + + Observer *Observer + Store *InputStore + Completion *Completion + + // The fields below are touched only by Reconcile; the controller runs one worker, so + // they need no locking. + + // finalFailures counts failed reconciles after the Sweeper exited. + finalFailures int +} + +func (r *Reconciler) key() types.NamespacedName { + return types.NamespacedName{Namespace: r.Namespace, Name: r.RunName} +} + +// Reconcile handles the single run key. Both the input observer and informer events +// enqueue it; the returned error makes the workqueue retry with backoff. +func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + if req.NamespacedName != r.key() || r.Completion.Finished() { + return ctrl.Result{}, nil + } + return ctrl.Result{}, r.reconcile(ctx) +} + +func (r *Reconciler) reconcile(ctx context.Context) error { + var run v1beta2.DynamoGraphDeploymentRun + if err := r.Client.Get(ctx, r.key(), &run); err != nil { + return fmt.Errorf("getting run: %w", err) + } + if string(run.UID) != r.RunUID { + // The run was deleted and recreated under the same name; this pod belongs to the + // previous incarnation and must not touch the new one. + r.Completion.Finish(fmt.Errorf("run %s has uid %s, this publisher serves uid %s", r.RunName, run.UID, r.RunUID)) + return nil + } + + // Read the container state BEFORE the snapshot: the Sweeper's last write precedes its + // exit, so an exit observed here means that write is visible to the check below. + state, stateErr := r.sweeperState(ctx) + if stateErr != nil { + ctrl.LoggerFrom(ctx).Error(stateErr, "reading sweeper container state") + } + exited := stateErr == nil && state.Exited + if exited { + r.Observer.Check() + } + + input := r.Store.Load() + if input.Err != nil && exited { + if errors.Is(input.Err, ErrProtocolViolation) { + r.Completion.Finish(input.Err) + return nil + } + return r.retryOrFail(exited, input.Err) + } + if input.Snapshot == nil { + if exited { + r.Completion.Finish(exitResult(state)) + } + return nil // the observer enqueues the key when the first snapshot arrives + } + + desired := Desire(r.RunName, r.RunUID, input.Snapshot) + if err := r.apply(ctx, &run, desired); err != nil { + return r.retryOrFail(exited, err) + } + switch { + case desired.Terminal: + r.Completion.Finish(terminalResult(desired.Phase)) + case exited: + r.Completion.Finish(exitResult(state)) + } + return nil +} + +// retryOrFail hands a failure to the workqueue's backoff, except where retrying cannot +// help (malformed projection) or the Sweeper is gone and the attempts are used up. +func (r *Reconciler) retryOrFail(exited bool, err error) error { + var inputErr *dgdcreconcile.DiffInputError + if errors.As(err, &inputErr) { + r.Completion.Finish(err) // retrying cannot fix it + return nil + } + if apierrors.IsConflict(err) { + // A write was based on a stale cached read. The informer delivers the newer + // version and the retry decides again, so this is not a failed attempt. + return err + } + if exited { + r.finalFailures++ + if r.finalFailures >= finalReconcileAttempts { + r.Completion.Finish(fmt.Errorf("reconciling last snapshot: %w", err)) + return nil + } + } + return err +} + +// apply drives the cluster toward the desired state: create missing candidates, finish +// the status of candidates that lack it, update the run, then delete obsolete candidates. +// It is idempotent and safe to retry; existing candidate specs are never touched. +func (r *Reconciler) apply(ctx context.Context, run *v1beta2.DynamoGraphDeploymentRun, desired DesiredState) error { + if desired.Failed > 0 { + ctrl.LoggerFrom(ctx).V(1).Info("candidates failed materialization and are not published", "count", desired.Failed) + } + var list v1beta2.DynamoGraphDeploymentCandidateList + if err := r.Client.List(ctx, &list, client.InNamespace(r.Namespace), client.MatchingLabels{LabelRunUID: r.RunUID}); err != nil { + return fmt.Errorf("listing candidates: %w", err) + } + current := make([]dgdcreconcile.CurrentDGDC, 0, len(list.Items)) + uids := make(map[string]types.UID, len(list.Items)) + versions := make(map[string]string, len(list.Items)) // cached resourceVersion per candidate + for i := range list.Items { + item := &list.Items[i] + current = append(current, dgdcreconcile.CurrentDGDC{ + Name: item.Name, + ID: item.Labels[LabelCandidateID], + Incomplete: item.Status.Metrics == nil, + }) + uids[item.Name] = item.UID + versions[item.Name] = item.ResourceVersion + } + actions, err := dgdcreconcile.ComputeActions(desired.Candidates, current) + if err != nil { + return err + } + // populated names the candidates whose status the API server already has; they are + // known from the live read that follows each apply, not from the cache. + populated := map[string]struct{}{} + for _, candidate := range actions.Creates { + name := desired.Names[candidate.ID] + stored, err := r.Writer.CreateCandidate(ctx, run, name, candidate) + if err != nil { + return fmt.Errorf("creating candidate %s: %w", candidate.ID, err) + } + versions[name] = stored.ResourceVersion // a new candidate is not in the cache; use the version the API server returned + if stored.Populated { + populated[name] = struct{}{} + } + } + unpopulated := make([]dgdcreconcile.DesiredCandidate, 0, len(actions.Creates)+len(actions.Incomplete)) + unpopulated = append(unpopulated, actions.Creates...) + unpopulated = append(unpopulated, actions.Incomplete...) + for _, candidate := range unpopulated { + name := desired.Names[candidate.ID] + if _, done := populated[name]; done { + continue + } + if err := r.Writer.PopulateStatus(ctx, name, versions[name], candidate); err != nil { + return fmt.Errorf("populating status of candidate %s: %w", candidate.ID, err) + } + } + + if err := r.updateRunStatus(ctx, run, desired.Status); err != nil { + return err + } + + for _, name := range actions.Deletes { + if err := r.Writer.DeleteCandidate(ctx, name, uids[name]); err != nil { + return fmt.Errorf("deleting candidate %s: %w", name, err) + } + } + return nil +} + +// updateRunStatus writes the run status when it differs from what the cache shows. A +// snapshot without a usable timestamp keeps the progress time the cached run already has, +// because a server-side-apply payload that omits the field would remove it. The write is +// constrained to the cached run's resourceVersion, so a decision made from a stale cache +// fails with a conflict instead of overwriting newer status. +func (r *Reconciler) updateRunStatus(ctx context.Context, run *v1beta2.DynamoGraphDeploymentRun, desired RunStatus) error { + observed := observedStatus(run) + if !desired.NeedsUpdate(observed) { + return nil + } + write := desired.WithPreservedProgress(observed.LastProgress) + if err := r.Writer.UpdateRunStatus(ctx, run.ResourceVersion, write); err != nil { + return fmt.Errorf("updating run status: %w", err) + } + return nil +} + +// observedStatus is the publisher-owned part of the run's current (cached) status. +func observedStatus(run *v1beta2.DynamoGraphDeploymentRun) RunStatus { + var observed RunStatus + if progress := run.Status.Progress; progress != nil && progress.Candidates != nil { + observed.Published = progress.Candidates.Published + } + if run.Status.LastProgressTime != nil { + observed.LastProgress = run.Status.LastProgressTime.UTC() + } + for _, ref := range run.Status.CandidateRefs { + observed.CandidateNames = append(observed.CandidateNames, ref.Name) + } + return observed +} + +// sweeperState reads the Sweeper container's status from this pod, via the cache. +func (r *Reconciler) sweeperState(ctx context.Context) (SweeperState, error) { + var pod corev1.Pod + if err := r.Client.Get(ctx, types.NamespacedName{Namespace: r.Namespace, Name: r.PodName}, &pod); err != nil { + return SweeperState{}, err + } + return sweeperStateOf(&pod, r.SweeperContainer), nil +} + +func sweeperStateOf(pod *corev1.Pod, container string) SweeperState { + for _, status := range pod.Status.ContainerStatuses { + if status.Name != container { + continue + } + if terminated := status.State.Terminated; terminated != nil { + return SweeperState{Exited: true, ExitCode: terminated.ExitCode} + } + return SweeperState{} + } + return SweeperState{} +} + +// decodeManifest extracts the DGD spec and the companion resources from a rendered +// manifest. A DGD in either supported apiVersion is converted to v1beta1, the same way +// the DGDR controller does it. +func decodeManifest(candidate dgdcreconcile.DesiredCandidate) (*v1beta1.DynamoGraphDeploymentSpec, string, error) { + manifest, err := ParseManifest(candidate.Spec) + if err != nil { + return nil, "", fmt.Errorf("candidate %s: %w", candidate.ID, err) + } + obj := &unstructured.Unstructured{Object: manifest.DGD} + var beta v1beta1.DynamoGraphDeployment + switch obj.GetAPIVersion() { + case dgdv1alpha1.GroupVersion.String(): + var alpha dgdv1alpha1.DynamoGraphDeployment + if err := runtime.DefaultUnstructuredConverter.FromUnstructured(obj.Object, &alpha); err != nil { + return nil, "", fmt.Errorf("candidate %s: %w", candidate.ID, err) + } + if err := alpha.ConvertTo(&beta); err != nil { + return nil, "", fmt.Errorf("candidate %s: %w", candidate.ID, err) + } + case v1beta1.GroupVersion.String(): + if err := runtime.DefaultUnstructuredConverter.FromUnstructured(obj.Object, &beta); err != nil { + return nil, "", fmt.Errorf("candidate %s: %w", candidate.ID, err) + } + default: + return nil, "", fmt.Errorf("candidate %s: unsupported DynamoGraphDeployment apiVersion %q", candidate.ID, obj.GetAPIVersion()) + } + return &beta.Spec, strings.Join(manifest.Companions, "\n---\n"), nil +} diff --git a/deploy/operator/internal/dgdrrunpublisher/reconciler_test.go b/deploy/operator/internal/dgdrrunpublisher/reconciler_test.go new file mode 100644 index 000000000000..27ff47cc988d --- /dev/null +++ b/deploy/operator/internal/dgdrrunpublisher/reconciler_test.go @@ -0,0 +1,900 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dgdrrunpublisher + +import ( + "context" + "encoding/json" + "errors" + "path/filepath" + "reflect" + "strings" + "testing" + "time" + + v1beta2 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta2" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/dgdcreconcile" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +const ( + testNamespace = "ns" + testRunName = "run" + testPodName = "pod-1" +) + +// memWriter is the reconciler's writer for tests. It logs every write attempt (a failed +// attempt is logged with a trailing "!") and, unless lag is set, applies the write to +// the fake client the reconciler reads from, which stands in for the informer cache. +// With lag set, writes are logged but never become visible to the reconciler, like a +// cache that has not caught up; the server map still records them. +type memWriter struct { + t *testing.T + c client.Client + ops []string + status []RunStatus + // statusVersions and populateVersions record the resourceVersion each write was + // constrained to. + statusVersions []string + populateVersions map[string]string + lag bool + // server is what the API server holds, whatever the cache shows: the candidates that + // exist, and whether each has its status. + server map[string]bool + // failOn makes the named operation ("create", "populate", "status", "delete") fail; + // once, unless sticky. + failOn map[string]error + sticky bool +} + +func (w *memWriter) fail(op string) error { + err, ok := w.failOn[op] + if !ok { + return nil + } + if !w.sticky { + delete(w.failOn, op) + } + return err +} + +func (w *memWriter) record(op string, err error) error { + if err != nil { + op += "!" + } + w.ops = append(w.ops, op) + return err +} + +func (w *memWriter) CreateCandidate(ctx context.Context, _ *v1beta2.DynamoGraphDeploymentRun, name string, candidate dgdcreconcile.DesiredCandidate) (StoredCandidate, error) { + if err := w.record("create:"+name, w.fail("create")); err != nil { + return StoredCandidate{}, err + } + // The apply is idempotent; the read-back reports what the server holds. + populated := w.server[name] + w.server[name] = populated + stored := StoredCandidate{ResourceVersion: createdVersion(name), Populated: populated} + if w.lag { + return stored, nil + } + object := &v1beta2.DynamoGraphDeploymentCandidate{ObjectMeta: metav1.ObjectMeta{ + Namespace: testNamespace, + Name: name, + UID: types.UID("uid-" + name), + Labels: map[string]string{LabelRunUID: testRunUID, LabelCandidateID: candidate.ID}, + }} + return stored, client.IgnoreAlreadyExists(w.c.Create(ctx, object)) +} + +// createdVersion is the resourceVersion the fake API server "returns" for a new candidate. +func createdVersion(name string) string { return "created-" + name } + +func (w *memWriter) PopulateStatus(ctx context.Context, name, resourceVersion string, _ dgdcreconcile.DesiredCandidate) error { + if err := w.record("populate:"+name, w.fail("populate")); err != nil { + return err + } + w.populateVersions[name] = resourceVersion + w.server[name] = true + if w.lag { + return nil + } + var object v1beta2.DynamoGraphDeploymentCandidate + if err := w.c.Get(ctx, types.NamespacedName{Namespace: testNamespace, Name: name}, &object); err != nil { + return err + } + object.Status.Metrics = &runtime.RawExtension{Raw: []byte("{}")} + return w.c.Update(ctx, &object) +} + +func (w *memWriter) UpdateRunStatus(ctx context.Context, resourceVersion string, status RunStatus) error { + if err := w.record("status", w.fail("status")); err != nil { + return err + } + w.status = append(w.status, status) + w.statusVersions = append(w.statusVersions, resourceVersion) + if w.lag { + return nil + } + var run v1beta2.DynamoGraphDeploymentRun + if err := w.c.Get(ctx, types.NamespacedName{Namespace: testNamespace, Name: testRunName}, &run); err != nil { + return err + } + run.Status.Progress = &v1beta2.DGDRRunProgress{Candidates: &v1beta2.CandidateProgress{Published: status.Published}} + if !status.LastProgress.IsZero() { + progressed := metav1.NewTime(status.LastProgress) + run.Status.LastProgressTime = &progressed + } + run.Status.CandidateRefs = nil + for _, name := range status.CandidateNames { + run.Status.CandidateRefs = append(run.Status.CandidateRefs, corev1.LocalObjectReference{Name: name}) + } + return w.c.Update(ctx, &run) +} + +func (w *memWriter) DeleteCandidate(ctx context.Context, name string, _ types.UID) error { + if err := w.record("delete:"+name, w.fail("delete")); err != nil { + return err + } + if w.lag { + return nil + } + return client.IgnoreNotFound(w.c.Delete(ctx, &v1beta2.DynamoGraphDeploymentCandidate{ObjectMeta: metav1.ObjectMeta{Namespace: testNamespace, Name: name}})) +} + +type fixture struct { + t *testing.T + r *Reconciler + w *memWriter + c client.Client +} + +func newFixture(t *testing.T) *fixture { + t.Helper() + scheme, err := NewScheme() + if err != nil { + t.Fatal(err) + } + run := &v1beta2.DynamoGraphDeploymentRun{ObjectMeta: metav1.ObjectMeta{Namespace: testNamespace, Name: testRunName, UID: testRunUID}} + pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Namespace: testNamespace, Name: testPodName}} + c := fake.NewClientBuilder().WithScheme(scheme).WithObjects(run, pod).Build() + store := &InputStore{} + observer := &Observer{ + Store: store, + Path: filepath.Join(t.TempDir(), SnapshotFileName), + MinInterval: time.Millisecond, + MaxInterval: time.Millisecond, + } + w := &memWriter{t: t, c: c, failOn: map[string]error{}, populateVersions: map[string]string{}, server: map[string]bool{}} + return &fixture{t: t, c: c, w: w, r: &Reconciler{ + Client: c, + Writer: w, + Namespace: testNamespace, + RunName: testRunName, + RunUID: testRunUID, + PodName: testPodName, + SweeperContainer: "sweeper", + Observer: observer, + Store: store, + Completion: NewCompletion(), + }} +} + +// publish writes a snapshot the way the Sweeper does and lets the observer see it. +func (f *fixture) publish(data []byte) { + f.t.Helper() + writeSnapshot(f.t, filepath.Dir(f.r.Observer.Path), data) + f.r.Observer.Check() +} + +func (f *fixture) reconcile() error { + f.t.Helper() + _, err := f.r.Reconcile(context.Background(), ctrl.Request{NamespacedName: types.NamespacedName{Namespace: testNamespace, Name: testRunName}}) + return err +} + +func (f *fixture) mustReconcile() { + f.t.Helper() + if err := f.reconcile(); err != nil { + f.t.Fatalf("reconcile: %v", err) + } +} + +// exitSweeper reports the Sweeper container as terminated, as the pod status does. +func (f *fixture) exitSweeper(code int32) { + f.t.Helper() + var pod corev1.Pod + if err := f.c.Get(context.Background(), types.NamespacedName{Namespace: testNamespace, Name: testPodName}, &pod); err != nil { + f.t.Fatal(err) + } + pod.Status.ContainerStatuses = []corev1.ContainerStatus{{Name: "sweeper", State: corev1.ContainerState{Terminated: &corev1.ContainerStateTerminated{ExitCode: code}}}} + if err := f.c.Status().Update(context.Background(), &pod); err != nil { + f.t.Fatal(err) + } +} + +// deleteCandidate removes a candidate behind the publisher's back, from the API server +// and from the cache. +func (f *fixture) deleteCandidate(name string) { + f.t.Helper() + delete(f.w.server, name) + object := &v1beta2.DynamoGraphDeploymentCandidate{ObjectMeta: metav1.ObjectMeta{Namespace: testNamespace, Name: name}} + if err := f.c.Delete(context.Background(), object); err != nil { + f.t.Fatal(err) + } +} + +func (f *fixture) takeOps() []string { + ops := f.w.ops + f.w.ops = nil + return ops +} + +func (f *fixture) lastStatus() RunStatus { + f.t.Helper() + if len(f.w.status) == 0 { + f.t.Fatal("no run status was written") + } + return f.w.status[len(f.w.status)-1] +} + +// withProgress rewrites a snapshot's progress counters. +func withProgress(t *testing.T, data []byte, round, evaluated int) []byte { + t.Helper() + doc := map[string]any{} + if err := json.Unmarshal(data, &doc); err != nil { + t.Fatal(err) + } + doc["progress"] = map[string]any{"round": round, "evaluated": evaluated} + out, err := json.Marshal(doc) + if err != nil { + t.Fatal(err) + } + return out +} + +func TestReconcileCreatesPopulatesAndReferencesCandidatesInOrder(t *testing.T) { + f := newFixture(t) + f.publish(snapshotJSON(t, PhaseRunning, cand{id: "a"}, cand{id: "b"})) + f.mustReconcile() + want := []string{"create:" + cn("a"), "create:" + cn("b"), "populate:" + cn("a"), "populate:" + cn("b"), "status"} + if got := f.takeOps(); !reflect.DeepEqual(got, want) { + t.Fatalf("ops = %v, want %v", got, want) + } + if got, want := f.lastStatus().CandidateNames, []string{cn("a"), cn("b")}; !reflect.DeepEqual(got, want) { + t.Fatalf("refs = %v, want %v", got, want) + } +} + +func TestSwapOnlyReordersTheReferences(t *testing.T) { + f := newFixture(t) + f.publish(snapshotJSON(t, PhaseRunning, cand{id: "a"}, cand{id: "b"})) + f.mustReconcile() + f.takeOps() + f.publish(snapshotJSON(t, PhaseRunning, cand{id: "b"}, cand{id: "a"})) + f.mustReconcile() + if got := f.takeOps(); !reflect.DeepEqual(got, []string{"status"}) { + t.Fatalf("a swap must not touch any candidate, ops = %v", got) + } + if got, want := f.lastStatus().CandidateNames, []string{cn("b"), cn("a")}; !reflect.DeepEqual(got, want) { + t.Fatalf("refs = %v, want %v", got, want) + } +} + +func TestReconcilingUnchangedStateWritesNothing(t *testing.T) { + f := newFixture(t) + f.publish(snapshotJSON(t, PhaseRunning, cand{id: "a"})) + f.mustReconcile() + f.takeOps() + f.mustReconcile() + f.mustReconcile() + if got := f.takeOps(); len(got) != 0 { + t.Fatalf("unchanged state must cause no writes, got %v", got) + } +} + +func TestCacheLagRepeatsOnlyIdempotentWrites(t *testing.T) { + f := newFixture(t) + f.w.lag = true // the cache never shows what was written + f.publish(snapshotJSON(t, PhaseRunning, cand{id: "a"})) + f.mustReconcile() + if got, want := f.takeOps(), []string{"create:" + cn("a"), "populate:" + cn("a"), "status"}; !reflect.DeepEqual(got, want) { + t.Fatalf("first reconcile ops = %v, want %v", got, want) + } + f.mustReconcile() + // The candidate apply repeats harmlessly and the server reports the candidate populated, + // so its status is not written a second time. The run status is written again, still + // constrained to the cached version: against a real API server the first write has + // moved the run on, so this one is rejected as stale and retried through the workqueue. + if got, want := f.takeOps(), []string{"create:" + cn("a"), "status"}; !reflect.DeepEqual(got, want) { + t.Fatalf("second reconcile ops = %v, want %v", got, want) + } + if len(f.w.statusVersions) != 2 || f.w.statusVersions[0] != f.w.statusVersions[1] { + t.Fatalf("both status writes must carry the cached run version, got %v", f.w.statusVersions) + } +} + +func TestExternallyDeletedCandidateIsRecreated(t *testing.T) { + f := newFixture(t) + f.publish(snapshotJSON(t, PhaseRunning, cand{id: "a"})) + f.mustReconcile() + f.takeOps() + f.deleteCandidate(cn("a")) + f.mustReconcile() + if got, want := f.takeOps(), []string{"create:" + cn("a"), "populate:" + cn("a")}; !reflect.DeepEqual(got, want) { + t.Fatalf("a candidate deleted behind the publisher's back must be repaired, ops = %v, want %v", got, want) + } +} + +func TestExternallyModifiedRunStatusIsRepaired(t *testing.T) { + f := newFixture(t) + f.publish(snapshotJSON(t, PhaseRunning, cand{id: "a"})) + f.mustReconcile() + f.takeOps() + var run v1beta2.DynamoGraphDeploymentRun + if err := f.c.Get(context.Background(), types.NamespacedName{Namespace: testNamespace, Name: testRunName}, &run); err != nil { + t.Fatal(err) + } + run.Status.CandidateRefs = []corev1.LocalObjectReference{{Name: "changed-by-someone-else"}} + if err := f.c.Update(context.Background(), &run); err != nil { + t.Fatal(err) + } + f.mustReconcile() + if got, want := f.takeOps(), []string{"status"}; !reflect.DeepEqual(got, want) { + t.Fatalf("a run status modified behind the publisher's back must be repaired, ops = %v, want %v", got, want) + } +} + +func TestPartialSuccessRetryRepeatsOnlyTheUnfinishedWrites(t *testing.T) { + f := newFixture(t) + f.w.failOn["status"] = errors.New("api unavailable") + f.publish(snapshotJSON(t, PhaseRunning, cand{id: "a"})) + if err := f.reconcile(); err == nil { + t.Fatal("a failed run status write must be returned for the workqueue to retry with backoff") + } + f.mustReconcile() // the retry, possibly triggered early by the echo of the candidate create + want := []string{"create:" + cn("a"), "populate:" + cn("a"), "status!", "status"} + if got := f.takeOps(); !reflect.DeepEqual(got, want) { + t.Fatalf("ops = %v, want %v", got, want) + } +} + +func TestCreateFailureSkipsTheRunStatusAndIsRetried(t *testing.T) { + f := newFixture(t) + f.w.failOn["create"] = errors.New("boom") + f.publish(snapshotJSON(t, PhaseRunning, cand{id: "a"})) + if err := f.reconcile(); err == nil { + t.Fatal("want the create failure returned") + } + for _, op := range f.takeOps() { + if op == "status" { + t.Fatal("the run must not reference a candidate that was not created") + } + } + f.mustReconcile() + if got := f.takeOps(); got[len(got)-1] != "status" { + t.Fatalf("ops after retry = %v", got) + } +} + +func TestDroppedCandidateIsDeletedAfterTheRunStatusUpdate(t *testing.T) { + f := newFixture(t) + f.publish(snapshotJSON(t, PhaseRunning, cand{id: "a"}, cand{id: "b"})) + f.mustReconcile() + f.takeOps() + f.publish(withProgress(t, snapshotJSON(t, PhaseRunning, cand{id: "a"}), 3, 9)) + f.mustReconcile() + if got, want := f.takeOps(), []string{"status", "delete:" + cn("b")}; !reflect.DeepEqual(got, want) { + t.Fatalf("ops = %v, want %v", got, want) + } +} + +func TestFailedMaterializationIsNotCreatedButCounted(t *testing.T) { + f := newFixture(t) + f.publish(snapshotJSON(t, PhaseRunning, cand{id: "a"}, cand{id: "x", failed: true})) + f.mustReconcile() + for _, op := range f.takeOps() { + if strings.Contains(op, "x") && strings.HasPrefix(op, "create") { + t.Fatalf("a failed candidate must not be created: %s", op) + } + } + status := f.lastStatus() + if !reflect.DeepEqual(status.CandidateNames, []string{cn("a")}) || status.Published != 1 { + t.Fatalf("status = %+v", status) + } +} + +func TestCandidateWithoutStatusIsCompletedNotRecreated(t *testing.T) { + f := newFixture(t) + existing := &v1beta2.DynamoGraphDeploymentCandidate{ObjectMeta: metav1.ObjectMeta{ + Namespace: testNamespace, + Name: cn("a"), + Labels: map[string]string{LabelRunUID: testRunUID, LabelCandidateID: "a"}, + }} + if err := f.c.Create(context.Background(), existing); err != nil { + t.Fatal(err) + } + f.publish(snapshotJSON(t, PhaseRunning, cand{id: "a"})) + f.mustReconcile() + if got, want := f.takeOps(), []string{"populate:" + cn("a"), "status"}; !reflect.DeepEqual(got, want) { + t.Fatalf("ops = %v, want %v", got, want) + } +} + +func TestProgressTimeSurvivesASnapshotWithoutATimestamp(t *testing.T) { + f := newFixture(t) + first := time.Date(2026, 10, 5, 21, 35, 12, 0, time.UTC) + f.publish(withTimestamp(t, snapshotJSON(t, PhaseRunning, cand{id: "a"}), "2026-10-05T21:35:12Z")) + f.mustReconcile() + f.publish(withTimestamp(t, snapshotJSON(t, PhaseRunning, cand{id: "a"}, cand{id: "b"}), "")) + f.mustReconcile() + if len(f.w.status) != 2 { + t.Fatalf("want two status writes, got %d", len(f.w.status)) + } + // The preserved time comes from the cached run, the current observed state; a stale + // cache would lose the version check instead of the timestamp. + if got := f.lastStatus().LastProgress; !got.Equal(first) { + t.Fatalf("the second write must repeat the observed progress time %v, got %v", first, got) + } +} + +func TestTerminalSucceededSnapshotIsReconciledThenAcknowledged(t *testing.T) { + f := newFixture(t) + f.publish(snapshotJSON(t, PhaseSucceeded, cand{id: "a"})) + f.mustReconcile() + if !f.r.Completion.Finished() || f.r.Completion.Err() != nil { + t.Fatalf("finished=%v err=%v", f.r.Completion.Finished(), f.r.Completion.Err()) + } + if len(f.w.status) != 1 { + t.Fatal("the terminal snapshot must be reconciled before the publisher exits") + } +} + +func TestTerminalFailedSnapshotIsReconciledThenFails(t *testing.T) { + f := newFixture(t) + f.publish(snapshotJSON(t, PhaseFailed, cand{id: "a"})) + f.mustReconcile() + if err := f.r.Completion.Err(); !errors.Is(err, ErrRunFailed) || ExitCode(err) != ExitRunFailed { + t.Fatalf("err = %v", err) + } + if len(f.w.status) != 1 { + t.Fatal("the failed snapshot must still be reconciled") + } +} + +func TestTerminalSnapshotReconcileFailureIsRetriedUntilItSucceeds(t *testing.T) { + f := newFixture(t) + f.w.failOn["status"] = errors.New("api unavailable") + f.publish(snapshotJSON(t, PhaseSucceeded, cand{id: "a"})) + if err := f.reconcile(); err == nil || f.r.Completion.Finished() { + t.Fatalf("a terminal snapshot that could not be reconciled must be retried, err=%v", err) + } + f.mustReconcile() + if !f.r.Completion.Finished() || f.r.Completion.Err() != nil { + t.Fatalf("finished=%v err=%v", f.r.Completion.Finished(), f.r.Completion.Err()) + } +} + +func TestSweeperCrashWithoutTerminalSnapshotReconcilesTheLastSnapshot(t *testing.T) { + f := newFixture(t) + f.publish(snapshotJSON(t, PhaseRunning, cand{id: "a"})) + f.exitSweeper(137) + f.mustReconcile() + if !f.r.Completion.Finished() || f.r.Completion.Err() != nil { + t.Fatalf("the Sweeper's own exit code fails the Job, not the publisher: finished=%v err=%v", f.r.Completion.Finished(), f.r.Completion.Err()) + } + if len(f.w.status) != 1 { + t.Fatal("the last snapshot must be reconciled before exit") + } +} + +func TestSweeperCleanExitWithoutTerminalSnapshotIsAProtocolViolation(t *testing.T) { + f := newFixture(t) + f.publish(snapshotJSON(t, PhaseRunning, cand{id: "a"})) + f.exitSweeper(0) + f.mustReconcile() + if err := f.r.Completion.Err(); !errors.Is(err, ErrMissingTerminalSnapshot) || ExitCode(err) != ExitProtocolViolation { + t.Fatalf("err = %v", err) + } + if len(f.w.status) != 1 { + t.Fatal("the last snapshot must be reconciled before exit") + } +} + +func TestSweeperExitBeforeAnySnapshot(t *testing.T) { + crash := newFixture(t) + crash.exitSweeper(1) + crash.mustReconcile() + if !crash.r.Completion.Finished() || crash.r.Completion.Err() != nil { + t.Fatalf("crash: finished=%v err=%v", crash.r.Completion.Finished(), crash.r.Completion.Err()) + } + clean := newFixture(t) + clean.exitSweeper(0) + clean.mustReconcile() + if err := clean.r.Completion.Err(); !errors.Is(err, ErrMissingTerminalSnapshot) { + t.Fatalf("clean exit with no snapshot at all: err = %v", err) + } +} + +func TestSnapshotWrittenJustBeforeTheSweeperExitIsReadAfterTheExitIsSeen(t *testing.T) { + f := newFixture(t) + f.publish(snapshotJSON(t, PhaseRunning, cand{id: "a"})) + f.mustReconcile() + // The Sweeper writes its terminal snapshot and exits; the observer has not looked yet. + writeSnapshot(t, filepath.Dir(f.r.Observer.Path), snapshotJSON(t, PhaseSucceeded, cand{id: "a"}, cand{id: "b"})) + f.exitSweeper(0) + f.mustReconcile() + if !f.r.Completion.Finished() || f.r.Completion.Err() != nil { + t.Fatalf("the terminal snapshot must be picked up on exit: finished=%v err=%v", f.r.Completion.Finished(), f.r.Completion.Err()) + } + if got := f.lastStatus().CandidateNames; len(got) != 2 { + t.Fatalf("refs = %v", got) + } +} + +func TestInvalidFinalSnapshotIsAProtocolViolationExit(t *testing.T) { + f := newFixture(t) + writeSnapshot(t, filepath.Dir(f.r.Observer.Path), []byte(`{"schemaVersion":2}`)) + f.exitSweeper(0) + f.mustReconcile() + if err := f.r.Completion.Err(); ExitCode(err) != ExitProtocolViolation { + t.Fatalf("err = %v, exit = %d", err, ExitCode(err)) + } +} + +func TestRegressingSnapshotIsNotReconciled(t *testing.T) { + f := newFixture(t) + f.publish(withProgress(t, snapshotJSON(t, PhaseRunning, cand{id: "a"}), 3, 5)) + f.mustReconcile() + f.takeOps() + f.publish(withProgress(t, snapshotJSON(t, PhaseRunning, cand{id: "a"}, cand{id: "b"}), 2, 99)) + f.mustReconcile() + if got := f.takeOps(); len(got) != 0 { + t.Fatalf("a regressing snapshot must not be reconciled, got %v", got) + } +} + +func TestFinalReconcileFailureEndsTheProcessAfterBoundedRetries(t *testing.T) { + f := newFixture(t) + f.w.failOn["status"] = errors.New("api unavailable") + f.w.sticky = true + f.publish(snapshotJSON(t, PhaseRunning, cand{id: "a"})) + f.exitSweeper(1) + for attempt := 1; attempt < finalReconcileAttempts; attempt++ { + if err := f.reconcile(); err == nil || f.r.Completion.Finished() { + t.Fatalf("attempt %d: want a retry, err=%v", attempt, err) + } + } + f.mustReconcile() + if err := f.r.Completion.Err(); err == nil || ExitCode(err) != ExitReconcileFailed { + t.Fatalf("err = %v", err) + } +} + +func TestMalformedProjectionFailsImmediately(t *testing.T) { + f := newFixture(t) + f.publish(snapshotJSON(t, PhaseRunning, cand{id: "a"})) + // Two live candidates claiming the same id make the diff input malformed. + for _, name := range []string{"x-1", "x-2"} { + object := &v1beta2.DynamoGraphDeploymentCandidate{ObjectMeta: metav1.ObjectMeta{ + Namespace: testNamespace, Name: name, + Labels: map[string]string{LabelRunUID: testRunUID, LabelCandidateID: "dup"}, + }} + if err := f.c.Create(context.Background(), object); err != nil { + t.Fatal(err) + } + } + f.mustReconcile() + var inputErr *dgdcreconcile.DiffInputError + if err := f.r.Completion.Err(); !errors.As(err, &inputErr) || ExitCode(err) != ExitReconcileFailed { + t.Fatalf("err = %v", err) + } +} + +func TestRunRecreatedUnderTheSameNameIsNotTouched(t *testing.T) { + f := newFixture(t) + var run v1beta2.DynamoGraphDeploymentRun + if err := f.c.Get(context.Background(), types.NamespacedName{Namespace: testNamespace, Name: testRunName}, &run); err != nil { + t.Fatal(err) + } + run.UID = "another-uid" + if err := f.c.Update(context.Background(), &run); err != nil { + t.Fatal(err) + } + f.publish(snapshotJSON(t, PhaseRunning, cand{id: "a"})) + f.mustReconcile() + if !f.r.Completion.Finished() || f.r.Completion.Err() == nil { + t.Fatal("a publisher serving a previous incarnation must stop") + } + if len(f.w.ops) != 0 { + t.Fatalf("it must not write anything: %v", f.w.ops) + } +} + +func TestReconcileIgnoresOtherKeysAndAFinishedRun(t *testing.T) { + f := newFixture(t) + f.publish(snapshotJSON(t, PhaseSucceeded, cand{id: "a"})) + if _, err := f.r.Reconcile(context.Background(), ctrl.Request{NamespacedName: types.NamespacedName{Namespace: testNamespace, Name: "other"}}); err != nil || len(f.w.ops) != 0 { + t.Fatalf("another key must be ignored: err=%v ops=%v", err, f.w.ops) + } + f.mustReconcile() + ops := len(f.w.ops) + f.mustReconcile() + if len(f.w.ops) != ops { + t.Fatal("a finished run must not be reconciled again") + } +} + +func podWith(statuses ...corev1.ContainerStatus) *corev1.Pod { + return &corev1.Pod{Status: corev1.PodStatus{ContainerStatuses: statuses}} +} + +func TestSweeperStateOfReadsOnlyTheSweeperContainer(t *testing.T) { + terminated := func(name string, code int32) corev1.ContainerStatus { + return corev1.ContainerStatus{Name: name, State: corev1.ContainerState{Terminated: &corev1.ContainerStateTerminated{ExitCode: code}}} + } + running := corev1.ContainerStatus{Name: "sweeper", State: corev1.ContainerState{Running: &corev1.ContainerStateRunning{}}} + + cases := map[string]struct { + pod *corev1.Pod + want SweeperState + }{ + "no statuses yet": {podWith(), SweeperState{}}, + "running": {podWith(running), SweeperState{}}, + "exited non-zero": {podWith(terminated("sweeper", 3)), SweeperState{Exited: true, ExitCode: 3}}, + "exited zero": {podWith(terminated("sweeper", 0)), SweeperState{Exited: true}}, + "other container only": {podWith(terminated("publisher", 1)), SweeperState{}}, + } + for name, tc := range cases { + if got := sweeperStateOf(tc.pod, "sweeper"); got != tc.want { + t.Errorf("%s: got %+v, want %+v", name, got, tc.want) + } + } +} + +func TestObservedStatusMirrorsTheRunStatus(t *testing.T) { + when := metav1.NewTime(time.Date(2026, 10, 5, 21, 35, 12, 0, time.UTC)) + run := &v1beta2.DynamoGraphDeploymentRun{Status: v1beta2.DynamoGraphDeploymentRunStatus{ + Progress: &v1beta2.DGDRRunProgress{Candidates: &v1beta2.CandidateProgress{Published: 2}}, + LastProgressTime: &when, + CandidateRefs: []corev1.LocalObjectReference{{Name: "x"}, {Name: "y"}}, + }} + want := RunStatus{Published: 2, LastProgress: when.Time, CandidateNames: []string{"x", "y"}} + if got := observedStatus(run); !reflect.DeepEqual(got, want) { + t.Fatalf("got %+v, want %+v", got, want) + } + if got := observedStatus(&v1beta2.DynamoGraphDeploymentRun{}); got.Published != 0 || !got.LastProgress.IsZero() { + t.Fatalf("an empty status must not look like reported progress: %+v", got) + } +} + +func TestCacheOptionsCoverOnlyThisRunItsCandidatesAndItsPod(t *testing.T) { + cfg := Config{Namespace: "ns", RunName: "run", RunUID: "uid-1", PodName: "pod-1"} + opts := cacheOptions(cfg) + if _, ok := opts.DefaultNamespaces["ns"]; !ok || len(opts.DefaultNamespaces) != 1 { + t.Fatalf("cache must be limited to the run's namespace: %v", opts.DefaultNamespaces) + } + if len(opts.ByObject) != 3 { + t.Fatalf("want selectors for the run, its candidates and the pod, got %d", len(opts.ByObject)) + } + for object, selector := range opts.ByObject { + switch object.(type) { + case *v1beta2.DynamoGraphDeploymentRun: + if selector.Field.String() != "metadata.name=run" { + t.Errorf("run selector = %q", selector.Field) + } + case *v1beta2.DynamoGraphDeploymentCandidate: + if selector.Label.String() != LabelRunUID+"=uid-1" { + t.Errorf("candidate selector = %q", selector.Label) + } + case *corev1.Pod: + if selector.Field.String() != "metadata.name=pod-1" { + t.Errorf("pod selector = %q", selector.Field) + } + default: + t.Errorf("unexpected cached type %T", object) + } + } +} + +func nested(t *testing.T, object *unstructured.Unstructured, fields ...string) (any, bool) { + t.Helper() + value, found, err := unstructured.NestedFieldNoCopy(object.Object, fields...) + if err != nil { + t.Fatal(err) + } + return value, found +} + +func TestCandidateApplyPayload(t *testing.T) { + scheme, err := NewScheme() + if err != nil { + t.Fatal(err) + } + run := &v1beta2.DynamoGraphDeploymentRun{ObjectMeta: metav1.ObjectMeta{Namespace: testNamespace, Name: testRunName, UID: testRunUID}} + candidate := dgdcreconcile.DesiredCandidate{ID: "a", Spec: validManifest, Parameters: map[string]any{"tp": 2}} + apply, err := buildCandidateApply(scheme, testNamespace, testRunUID, run, cn("a"), candidate) + if err != nil { + t.Fatal(err) + } + if apply.GetAPIVersion() != "nvidia.com/v1beta2" || apply.GetKind() != "DynamoGraphDeploymentCandidate" || apply.GetName() != cn("a") || apply.GetNamespace() != testNamespace { + t.Fatalf("identity = %s %s %s/%s", apply.GetAPIVersion(), apply.GetKind(), apply.GetNamespace(), apply.GetName()) + } + if _, found := nested(t, apply, "status"); found { + t.Fatal("status goes through the status subresource") + } + if _, found := nested(t, apply, "metadata", "creationTimestamp"); found { + t.Fatal("creationTimestamp must not be part of an apply payload") + } + if got := apply.GetLabels(); got[LabelRunUID] != testRunUID || got[LabelCandidateID] != "a" { + t.Fatalf("labels = %v", got) + } + if _, found := nested(t, apply, "spec", "parameters"); !found { + t.Fatal("the resolved parameters belong in the spec") + } + owners := apply.GetOwnerReferences() + if len(owners) != 1 || owners[0].UID != testRunUID || owners[0].Controller == nil || !*owners[0].Controller { + t.Fatalf("owner references = %+v", owners) + } +} + +func TestCandidateStatusApplyPayloadIsDeterministic(t *testing.T) { + candidate := dgdcreconcile.DesiredCandidate{ID: "a", Metrics: map[string]any{"score": 2}} + first, err := buildCandidateStatusApply(testNamespace, cn("a"), "17", candidate) + if err != nil { + t.Fatal(err) + } + second, err := buildCandidateStatusApply(testNamespace, cn("a"), "17", candidate) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(first.Object, second.Object) { + t.Fatal("the same input must produce the same payload") + } + if got, _ := nested(t, first, "metadata", "resourceVersion"); got != "17" { + t.Fatalf("the status write must be constrained to the version it was decided on, got %v", got) + } + if _, found := nested(t, first, "status", "metrics"); !found { + t.Fatal("the metrics are the candidate's status") + } + if _, found := nested(t, first, "status", "conditions"); found { + t.Fatal("candidate status is metrics only") + } +} + +func TestCandidateStatusApplyAlwaysWritesMetricsEvenWhenEmpty(t *testing.T) { + apply, err := buildCandidateStatusApply(testNamespace, cn("a"), "17", dgdcreconcile.DesiredCandidate{ID: "a"}) + if err != nil { + t.Fatal(err) + } + metrics, found := nested(t, apply, "status", "metrics") + if !found { + t.Fatal("an evaluated candidate without metrics must still carry an empty metrics object: its presence marks it populated") + } + if object, ok := metrics.(map[string]any); !ok || len(object) != 0 { + t.Fatalf("metrics = %#v, want an empty object", metrics) + } +} + +func TestRunStatusApplyPayload(t *testing.T) { + when := time.Date(2026, 10, 5, 21, 35, 12, 0, time.UTC) + status := RunStatus{Published: 2, LastProgress: when, CandidateNames: []string{"x", "y"}} + apply := buildRunStatusApply(testNamespace, testRunName, "42", status) + if apply.GetKind() != "DynamoGraphDeploymentRun" || apply.GetName() != testRunName { + t.Fatalf("identity = %s %s", apply.GetKind(), apply.GetName()) + } + if got, _ := nested(t, apply, "metadata", "resourceVersion"); got != "42" { + t.Fatalf("the status write must be constrained to the observed run version, got %v", got) + } + if _, found := nested(t, buildRunStatusApply(testNamespace, testRunName, "", status), "metadata", "resourceVersion"); found { + t.Fatal("no version known, no constraint") + } + if got, _ := nested(t, apply, "status", "lastProgressTime"); got != "2026-10-05T21:35:12Z" { + t.Fatalf("lastProgressTime = %v", got) + } + if got, _ := nested(t, apply, "status", "progress", "candidates", "published"); got != int64(2) { + t.Fatalf("published = %#v", got) + } + for _, field := range []string{"message", "provenance"} { + if _, found := nested(t, apply, "status", field); found { + t.Fatalf("status.%s does not exist in the API and must not be written", field) + } + } + refs, _ := nested(t, apply, "status", "candidateRefs") + if list, ok := refs.([]any); !ok || len(list) != 2 { + t.Fatalf("candidateRefs = %v", refs) + } + // A status without a progress time leaves the field out; the reconciler is what + // carries the known time into the status it writes. + status.LastProgress = time.Time{} + if _, found := nested(t, buildRunStatusApply(testNamespace, testRunName, "42", status), "status", "lastProgressTime"); found { + t.Fatal("no time known, so none is written") + } +} + +func TestRunStatusWriteIsConstrainedToTheObservedRunVersion(t *testing.T) { + f := newFixture(t) + var run v1beta2.DynamoGraphDeploymentRun + if err := f.c.Get(context.Background(), types.NamespacedName{Namespace: testNamespace, Name: testRunName}, &run); err != nil { + t.Fatal(err) + } + if run.ResourceVersion == "" { + t.Fatal("the fake client should assign a resourceVersion") + } + f.publish(snapshotJSON(t, PhaseRunning, cand{id: "a"})) + f.mustReconcile() + if len(f.w.statusVersions) != 1 || f.w.statusVersions[0] != run.ResourceVersion { + t.Fatalf("status versions = %v, want [%s]", f.w.statusVersions, run.ResourceVersion) + } +} + +func TestNewCandidateStatusUsesTheVersionReturnedByTheCreate(t *testing.T) { + f := newFixture(t) + f.publish(snapshotJSON(t, PhaseRunning, cand{id: "a"})) + f.mustReconcile() + if got, want := f.w.populateVersions[cn("a")], createdVersion(cn("a")); got != want { + t.Fatalf("populate version = %q, want the version from the create (%q)", got, want) + } +} + +func TestExistingCandidateStatusUsesItsCachedVersion(t *testing.T) { + f := newFixture(t) + existing := &v1beta2.DynamoGraphDeploymentCandidate{ObjectMeta: metav1.ObjectMeta{ + Namespace: testNamespace, + Name: cn("a"), + Labels: map[string]string{LabelRunUID: testRunUID, LabelCandidateID: "a"}, + }} + if err := f.c.Create(context.Background(), existing); err != nil { + t.Fatal(err) + } + var cached v1beta2.DynamoGraphDeploymentCandidate + if err := f.c.Get(context.Background(), types.NamespacedName{Namespace: testNamespace, Name: cn("a")}, &cached); err != nil { + t.Fatal(err) + } + f.publish(snapshotJSON(t, PhaseRunning, cand{id: "a"})) + f.mustReconcile() + if got := f.w.populateVersions[cn("a")]; got == "" || got != cached.ResourceVersion { + t.Fatalf("populate version = %q, want the cached version %q", got, cached.ResourceVersion) + } +} + +func TestConflictIsRetriedWithoutUsingUpTheFinalAttempts(t *testing.T) { + f := newFixture(t) + f.w.failOn["status"] = apierrors.NewConflict(schema.GroupResource{Group: "nvidia.com", Resource: "dynamographdeploymentruns"}, testRunName, errors.New("stale")) + f.w.sticky = true + f.publish(snapshotJSON(t, PhaseRunning, cand{id: "a"})) + f.exitSweeper(1) + for attempt := 0; attempt < 2*finalReconcileAttempts; attempt++ { + if err := f.reconcile(); !apierrors.IsConflict(err) || f.r.Completion.Finished() { + t.Fatalf("attempt %d: a conflict from a stale read must be retried, err=%v finished=%v", attempt, err, f.r.Completion.Finished()) + } + } + delete(f.w.failOn, "status") // the cache caught up + f.mustReconcile() + if !f.r.Completion.Finished() || f.r.Completion.Err() != nil { + t.Fatalf("finished=%v err=%v", f.r.Completion.Finished(), f.r.Completion.Err()) + } +} diff --git a/deploy/operator/internal/dgdrrunpublisher/snapshot.go b/deploy/operator/internal/dgdrrunpublisher/snapshot.go new file mode 100644 index 000000000000..5bbe05125bbd --- /dev/null +++ b/deploy/operator/internal/dgdrrunpublisher/snapshot.go @@ -0,0 +1,201 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dgdrrunpublisher + +import ( + "errors" + "fmt" + "regexp" + "strings" + + "sigs.k8s.io/yaml" +) + +// ErrProtocolViolation means the Sweeper broke the snapshot contract: a snapshot that +// cannot be decoded or validated, one that moves progress backwards, or a missing +// terminal snapshot. It maps to ExitProtocolViolation. +var ErrProtocolViolation = errors.New("snapshot protocol violation") + +func violation(format string, args ...any) error { + return fmt.Errorf("%w: %s", ErrProtocolViolation, fmt.Sprintf(format, args...)) +} + +// maxCandidateIDLength keeps ids well inside the 63-character Kubernetes label-value +// limit, since the id is recorded verbatim as a label on the candidate. +const maxCandidateIDLength = 48 + +// candidateIDPattern is a DNS-1123 label, valid both as a label value and inside an +// object name. +var candidateIDPattern = regexp.MustCompile(`^[a-z0-9]([-a-z0-9]*[a-z0-9])?$`) + +const dgdKind = "DynamoGraphDeployment" + +// supportedDGDAPIVersions are the DGD versions the candidate path can convert to v1beta1. +var supportedDGDAPIVersions = map[string]bool{ + "nvidia.com/v1alpha1": true, + "nvidia.com/v1beta1": true, +} + +// Manifest is one rendered candidate: exactly one DynamoGraphDeployment plus any +// companion resources (for example generated ConfigMaps) the renderer emitted with it. +type Manifest struct { + DGD map[string]any + Companions []string // raw YAML documents, in order +} + +var documentSeparator = regexp.MustCompile(`(?m)^---[ \t]*$`) + +// ParseManifest splits a multi-document manifest and locates its single +// DynamoGraphDeployment, which must carry a spec. +func ParseManifest(manifest string) (*Manifest, error) { + var out Manifest + for _, doc := range documentSeparator.Split(manifest, -1) { + if strings.TrimSpace(doc) == "" { + continue + } + var obj map[string]any + if err := yaml.Unmarshal([]byte(doc), &obj); err != nil { + return nil, fmt.Errorf("manifest document is not a YAML mapping: %w", err) + } + if len(obj) == 0 { + continue // comment-only document + } + if obj["kind"] != dgdKind { + out.Companions = append(out.Companions, strings.TrimSpace(doc)) + continue + } + if out.DGD != nil { + return nil, errors.New("manifest has more than one DynamoGraphDeployment") + } + if version, _ := obj["apiVersion"].(string); !supportedDGDAPIVersions[version] { + return nil, fmt.Errorf("DynamoGraphDeployment has unsupported apiVersion %q", version) + } + if _, ok := obj["spec"].(map[string]any); !ok { + return nil, errors.New("DynamoGraphDeployment has no spec") + } + out.DGD = obj + } + if out.DGD == nil { + return nil, errors.New("manifest has no DynamoGraphDeployment") + } + return &out, nil +} + +func validateManifest(id, manifest string) error { + if _, err := ParseManifest(manifest); err != nil { + return violation("manifest of candidate %q: %v", id, err) + } + return nil +} + +// SnapshotSchemaVersion is the only wire version this publisher understands. +const SnapshotSchemaVersion = 1 + +// Candidate outcomes carried by the snapshot. +const ( + OutcomeMaterialized = "materialized" + OutcomeMaterializationFailed = "materialization_failed" +) + +// Run phases carried by the snapshot. +const ( + PhaseRunning = "Running" + PhaseSucceeded = "Succeeded" + PhaseFailed = "Failed" +) + +// Snapshot is the complete current projection of one run, written atomically by the +// Sweeper container (see dynamo.aisimulate.output.dgdr_run.snapshot). It is desired +// state, not a command list: each snapshot supersedes the previous one entirely. +type Snapshot struct { + SchemaVersion int `json:"schemaVersion"` + Timestamp string `json:"timestamp,omitempty"` + Run SnapshotRun `json:"run"` + Progress SnapshotProgress `json:"progress"` + Candidates []SnapshotCandidate `json:"candidates"` +} + +type SnapshotRun struct { + Phase string `json:"phase"` + Terminal bool `json:"terminal"` + Message string `json:"message,omitempty"` + Error string `json:"error,omitempty"` +} + +type SnapshotProgress struct { + Round int32 `json:"round"` + Evaluated int32 `json:"evaluated"` +} + +// SnapshotCandidate is one evaluated point. Candidates are ordered best-first for +// scalar searches; the id never depends on that order. +type SnapshotCandidate struct { + ID string `json:"id"` + Outcome string `json:"outcome"` + Parameters map[string]any `json:"parameters,omitempty"` + Metrics map[string]any `json:"metrics,omitempty"` + Manifest string `json:"manifest,omitempty"` + Error string `json:"error,omitempty"` +} + +func ParseSnapshot(data []byte) (*Snapshot, error) { + var snap Snapshot + if err := yaml.Unmarshal(data, &snap); err != nil { + return nil, violation("decoding snapshot: %v", err) + } + if snap.SchemaVersion != SnapshotSchemaVersion { + return nil, violation("unsupported snapshot schemaVersion %d (want %d)", snap.SchemaVersion, SnapshotSchemaVersion) + } + switch snap.Run.Phase { + case PhaseRunning, PhaseSucceeded, PhaseFailed: + default: + return nil, violation("unknown run phase %q", snap.Run.Phase) + } + if snap.Run.Terminal != (snap.Run.Phase != PhaseRunning) { + return nil, violation("run phase %q inconsistent with terminal=%t", snap.Run.Phase, snap.Run.Terminal) + } + seen := make(map[string]struct{}, len(snap.Candidates)) + for i, candidate := range snap.Candidates { + if candidate.ID == "" { + return nil, violation("candidate %d has an empty id", i) + } + if len(candidate.ID) > maxCandidateIDLength || !candidateIDPattern.MatchString(candidate.ID) { + return nil, violation("candidate id %q is not a lowercase DNS label of at most %d characters", candidate.ID, maxCandidateIDLength) + } + if _, dup := seen[candidate.ID]; dup { + return nil, violation("duplicate candidate id %q", candidate.ID) + } + seen[candidate.ID] = struct{}{} + switch candidate.Outcome { + case OutcomeMaterialized: + if candidate.Manifest == "" { + return nil, violation("materialized candidate %q has no manifest", candidate.ID) + } + if err := validateManifest(candidate.ID, candidate.Manifest); err != nil { + return nil, err + } + case OutcomeMaterializationFailed: + if candidate.Error == "" { + return nil, violation("failed candidate %q has no error", candidate.ID) + } + default: + return nil, violation("candidate %q has unknown outcome %q", candidate.ID, candidate.Outcome) + } + } + return &snap, nil +} diff --git a/deploy/operator/internal/dgdrrunpublisher/snapshot_test.go b/deploy/operator/internal/dgdrrunpublisher/snapshot_test.go new file mode 100644 index 000000000000..3be65aa92092 --- /dev/null +++ b/deploy/operator/internal/dgdrrunpublisher/snapshot_test.go @@ -0,0 +1,158 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dgdrrunpublisher + +import ( + "encoding/json" + "errors" + "os" + "path/filepath" + "strings" + "testing" +) + +const validManifest = "apiVersion: nvidia.com/v1beta1\nkind: DynamoGraphDeployment\nspec: {}\n" + +const testRunUID = "uid-1" + +func cn(id string) string { return CandidateName("run", testRunUID, id) } + +type cand struct { + id string + failed bool + metrics map[string]any +} + +// snapshotJSON renders a snapshot (JSON is valid YAML, which the real parser accepts). +func snapshotJSON(t *testing.T, phase string, cands ...cand) []byte { + t.Helper() + var list []map[string]any + for _, c := range cands { + entry := map[string]any{"id": c.id, "parameters": map[string]any{"tp": 2}, "metrics": c.metrics} + if c.failed { + entry["outcome"] = OutcomeMaterializationFailed + entry["error"] = "boom" + } else { + entry["outcome"] = OutcomeMaterialized + entry["manifest"] = validManifest + } + list = append(list, entry) + } + data, err := json.Marshal(map[string]any{ + "schemaVersion": 1, + "run": map[string]any{"phase": phase, "terminal": phase != PhaseRunning}, + "progress": map[string]any{"round": 3, "evaluated": len(cands)}, + "candidates": list, + }) + if err != nil { + t.Fatal(err) + } + return data +} + +func writeSnapshot(t *testing.T, dir string, data []byte) { + t.Helper() + if err := os.WriteFile(filepath.Join(dir, SnapshotFileName), data, 0o600); err != nil { + t.Fatal(err) + } +} + +func mustParse(t *testing.T, data []byte) *Snapshot { + t.Helper() + snap, err := ParseSnapshot(data) + if err != nil { + t.Fatal(err) + } + return snap +} + +func withTimestamp(t *testing.T, data []byte, timestamp string) []byte { + t.Helper() + var doc map[string]any + if err := json.Unmarshal(data, &doc); err != nil { + t.Fatal(err) + } + doc["timestamp"] = timestamp + out, err := json.Marshal(doc) + if err != nil { + t.Fatal(err) + } + return out +} + +func TestParseSnapshotValidation(t *testing.T) { + bad := map[string]string{ + "schema": `{"schemaVersion":2,"run":{"phase":"Running","terminal":false}}`, + "phase": `{"schemaVersion":1,"run":{"phase":"Weird","terminal":false}}`, + "terminal flag": `{"schemaVersion":1,"run":{"phase":"Succeeded","terminal":false}}`, + "duplicate": `{"schemaVersion":1,"run":{"phase":"Running","terminal":false},"candidates":[{"id":"a","outcome":"materialized","manifest":"apiVersion: nvidia.com/v1beta1\\nkind: DynamoGraphDeployment\\nspec: {}\\n"},{"id":"a","outcome":"materialized","manifest":"apiVersion: nvidia.com/v1beta1\\nkind: DynamoGraphDeployment\\nspec: {}\\n"}]}`, + "no manifest": `{"schemaVersion":1,"run":{"phase":"Running","terminal":false},"candidates":[{"id":"a","outcome":"materialized"}]}`, + "no error": `{"schemaVersion":1,"run":{"phase":"Running","terminal":false},"candidates":[{"id":"a","outcome":"materialization_failed"}]}`, + "unsafe id": `{"schemaVersion":1,"run":{"phase":"Running","terminal":false},"candidates":[{"id":"Not_Valid","outcome":"materialized","manifest":"apiVersion: nvidia.com/v1beta1\\nkind: DynamoGraphDeployment\\nspec: {}\\n"}]}`, + "long id": `{"schemaVersion":1,"run":{"phase":"Running","terminal":false},"candidates":[{"id":"%s","outcome":"materialized","manifest":"apiVersion: nvidia.com/v1beta1\\nkind: DynamoGraphDeployment\\nspec: {}\\n"}]}`, + "wrong kind": `{"schemaVersion":1,"run":{"phase":"Running","terminal":false},"candidates":[{"id":"a","outcome":"materialized","manifest":"kind: ConfigMap\\nspec: {}\\n"}]}`, + "empty id": `{"schemaVersion":1,"run":{"phase":"Running","terminal":false},"candidates":[{"id":"","outcome":"materialized","manifest":"apiVersion: nvidia.com/v1beta1\\nkind: DynamoGraphDeployment\\nspec: {}\\n"}]}`, + } + bad["long id"] = strings.Replace(bad["long id"], "%s", strings.Repeat("a", 64), 1) + for name, doc := range bad { + if _, err := ParseSnapshot([]byte(doc)); !errors.Is(err, ErrProtocolViolation) { + t.Errorf("%s: want protocol violation, got %v", name, err) + } + } +} + +func TestParseManifestHandlesCompanionResources(t *testing.T) { + dgd := "apiVersion: nvidia.com/v1beta1\nkind: DynamoGraphDeployment\nspec: {}\n" + cm := "apiVersion: v1\nkind: ConfigMap\nmetadata: {}\n" + for name, manifest := range map[string]string{ + "dgd first": dgd + "---\n" + cm, + "configmap first": "# generated\n---\n" + cm + "---\n" + dgd, + } { + got, err := ParseManifest(manifest) + if err != nil { + t.Fatalf("%s: %v", name, err) + } + if got.DGD["kind"] != "DynamoGraphDeployment" || len(got.Companions) != 1 || !strings.Contains(got.Companions[0], "ConfigMap") { + t.Fatalf("%s: %+v", name, got) + } + } + for name, manifest := range map[string]string{ + "two dgds": dgd + "---\n" + dgd, + "no dgd": cm, + "no spec": "apiVersion: nvidia.com/v1beta1\nkind: DynamoGraphDeployment\n", + "no apiVersion": "kind: DynamoGraphDeployment\nspec: {}\n", + "unsupported apiVersion": "apiVersion: nvidia.com/v9\nkind: DynamoGraphDeployment\nspec: {}\n", + } { + if _, err := ParseManifest(manifest); err == nil { + t.Errorf("%s: want error", name) + } + } +} +func writeProgress(t *testing.T, dir, phase string, round, evaluated int) { + t.Helper() + doc := map[string]any{} + if err := json.Unmarshal(snapshotJSON(t, phase, cand{id: "a"}), &doc); err != nil { + t.Fatal(err) + } + doc["progress"] = map[string]any{"round": round, "evaluated": evaluated} + data, err := json.Marshal(doc) + if err != nil { + t.Fatal(err) + } + writeSnapshot(t, dir, data) +} diff --git a/deploy/operator/internal/dgdrrunpublisher/writer.go b/deploy/operator/internal/dgdrrunpublisher/writer.go new file mode 100644 index 000000000000..8a02f1c61934 --- /dev/null +++ b/deploy/operator/internal/dgdrrunpublisher/writer.go @@ -0,0 +1,235 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dgdrrunpublisher + +import ( + "context" + "encoding/json" + "time" + + v1beta2 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta2" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/dgdcreconcile" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" +) + +// StoredCandidate is the API server's view of a candidate right after it was applied. +type StoredCandidate struct { + // ResourceVersion is the version a status write for this candidate is constrained to. + ResourceVersion string + // Populated reports that the candidate already has its evaluation status (a metrics + // object, possibly empty), for example when a stale cache led the reconciler to apply a + // candidate that exists. + Populated bool +} + +// writer is every write the reconciler makes to the API server. It is a seam so the +// reconciler's behavior can be tested without server-side apply support in a fake client. +type writer interface { + // CreateCandidate applies the candidate and returns the API server's own copy of it, + // read after the apply. It must be idempotent: applying a candidate that already + // exists changes nothing. + CreateCandidate(ctx context.Context, run *v1beta2.DynamoGraphDeploymentRun, name string, candidate dgdcreconcile.DesiredCandidate) (StoredCandidate, error) + // PopulateStatus records the candidate's evaluation metrics. The + // write is constrained to resourceVersion, so a stale view of the candidate fails + // with a conflict instead of rewriting status that was already populated. + PopulateStatus(ctx context.Context, name, resourceVersion string, candidate dgdcreconcile.DesiredCandidate) error + // UpdateRunStatus writes the publisher-owned run status fields, constrained to the + // resourceVersion of the run the decision to write was based on. + UpdateRunStatus(ctx context.Context, resourceVersion string, status RunStatus) error + // DeleteCandidate deletes the candidate with the given UID precondition. + DeleteCandidate(ctx context.Context, name string, uid types.UID) error +} + +// clientWriter writes with server-side apply through a controller-runtime client. +type clientWriter struct { + Client client.Client + // Reader reads straight from the API server. A candidate's resourceVersion right + // after its creation cannot come from the informer cache, which has not seen it yet. + Reader client.Reader + Namespace string + RunName string + RunUID string +} + +var _ writer = (*clientWriter)(nil) + +func (w *clientWriter) apply(ctx context.Context, object *unstructured.Unstructured, status bool) error { + configuration := client.ApplyConfigurationFromUnstructured(object) + if status { + return w.Client.Status().Apply(ctx, configuration, client.FieldOwner(fieldOwner), client.ForceOwnership) + } + return w.Client.Apply(ctx, configuration, client.FieldOwner(fieldOwner), client.ForceOwnership) +} + +// CreateCandidate applies the candidate and reads it back from the API server. It is +// only called for a candidate missing from the cache, and re-applying the same content +// to a candidate that exists is a no-op, so no spec comparison is needed. The read-back +// tells the caller whether the status is populated already: the cache cannot, because it +// is exactly what may be stale. +func (w *clientWriter) CreateCandidate(ctx context.Context, run *v1beta2.DynamoGraphDeploymentRun, name string, candidate dgdcreconcile.DesiredCandidate) (StoredCandidate, error) { + object, err := buildCandidateApply(w.Client.Scheme(), w.Namespace, w.RunUID, run, name, candidate) + if err != nil { + return StoredCandidate{}, err + } + if err := w.apply(ctx, object, false); err != nil { + return StoredCandidate{}, err + } + var stored v1beta2.DynamoGraphDeploymentCandidate + if err := w.Reader.Get(ctx, types.NamespacedName{Namespace: w.Namespace, Name: name}, &stored); err != nil { + return StoredCandidate{}, err + } + return StoredCandidate{ + ResourceVersion: stored.ResourceVersion, + Populated: stored.Status.Metrics != nil, + }, nil +} + +func (w *clientWriter) PopulateStatus(ctx context.Context, name, resourceVersion string, candidate dgdcreconcile.DesiredCandidate) error { + object, err := buildCandidateStatusApply(w.Namespace, name, resourceVersion, candidate) + if err != nil { + return err + } + return w.apply(ctx, object, true) +} + +func (w *clientWriter) UpdateRunStatus(ctx context.Context, resourceVersion string, status RunStatus) error { + return w.apply(ctx, buildRunStatusApply(w.Namespace, w.RunName, resourceVersion, status), true) +} + +func (w *clientWriter) DeleteCandidate(ctx context.Context, name string, uid types.UID) error { + object := &v1beta2.DynamoGraphDeploymentCandidate{ObjectMeta: metav1.ObjectMeta{Namespace: w.Namespace, Name: name}} + var options []client.DeleteOption + if uid != "" { + options = append(options, client.Preconditions{UID: &uid}) + } + return client.IgnoreNotFound(w.Client.Delete(ctx, object, options...)) +} + +// buildCandidateApply is the server-side-apply payload that creates a candidate: labels +// and controller owner reference, the DGD spec with the resolved parameters, and the +// companion resources as an annotation. The status goes through the status subresource. +func buildCandidateApply(scheme *runtime.Scheme, namespace, runUID string, run *v1beta2.DynamoGraphDeploymentRun, name string, candidate dgdcreconcile.DesiredCandidate) (*unstructured.Unstructured, error) { + spec, companions, err := decodeManifest(candidate) + if err != nil { + return nil, err + } + parameters, err := normalizeJSON(candidate.Parameters) + if err != nil { + return nil, err + } + object := &v1beta2.DynamoGraphDeploymentCandidate{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace, + Name: name, + Labels: map[string]string{LabelRunUID: runUID, LabelCandidateID: candidate.ID}, + }, + Spec: v1beta2.DynamoGraphDeploymentCandidateSpec{DynamoGraphDeploymentSpec: *spec}, + } + if companions != "" { + object.Annotations = map[string]string{AnnotationAdditionalResources: companions} + } + if err := controllerutil.SetControllerReference(run, object, scheme); err != nil { + return nil, err + } + content, err := runtime.DefaultUnstructuredConverter.ToUnstructured(object) + if err != nil { + return nil, err + } + delete(content, "status") + unstructured.RemoveNestedField(content, "metadata", "creationTimestamp") + if err := unstructured.SetNestedField(content, parameters, "spec", "parameters"); err != nil { + return nil, err + } + apply := &unstructured.Unstructured{Object: content} + apply.SetGroupVersionKind(v1beta2.DynamoGraphDeploymentCandidateGVK) + return apply, nil +} + +// withResourceVersion constrains a server-side apply to one version of the object: the +// API server rejects it with a conflict if the object has changed since. +func withResourceVersion(metadata map[string]any, resourceVersion string) map[string]any { + if resourceVersion != "" { + metadata["resourceVersion"] = resourceVersion + } + return metadata +} + +// buildCandidateStatusApply is the status payload of a candidate: the metrics. Metrics are +// always written, even when empty, because their presence is what marks the candidate as +// populated. Candidate status is written once and then stays immutable, so the payload +// carries the resourceVersion the "not populated yet" decision was based on. +func buildCandidateStatusApply(namespace, name, resourceVersion string, candidate dgdcreconcile.DesiredCandidate) (*unstructured.Unstructured, error) { + metrics, err := normalizeJSON(candidate.Metrics) + if err != nil { + return nil, err + } + apply := &unstructured.Unstructured{Object: map[string]any{ + "metadata": withResourceVersion(map[string]any{"namespace": namespace, "name": name}, resourceVersion), + "status": map[string]any{"metrics": metrics}, + }} + apply.SetGroupVersionKind(v1beta2.DynamoGraphDeploymentCandidateGVK) + return apply, nil +} + +// buildRunStatusApply is the status payload of the run: the candidate references, the +// published-candidate count and the progress time. It lists every publisher-owned +// field it wants to keep: under server-side apply a field this manager owns and omits +// from the payload is removed. It carries the resourceVersion of the run the decision to +// write was based on, so a decision made from a stale cached run conflicts instead of +// overwriting a newer status. +func buildRunStatusApply(namespace, runName, resourceVersion string, status RunStatus) *unstructured.Unstructured { + refs := make([]any, 0, len(status.CandidateNames)) + for _, name := range status.CandidateNames { + refs = append(refs, map[string]any{"name": name}) + } + fields := map[string]any{ + "candidateRefs": refs, + "progress": map[string]any{"candidates": map[string]any{"published": int64(status.Published)}}, + } + if !status.LastProgress.IsZero() { + fields["lastProgressTime"] = status.LastProgress.UTC().Format(time.RFC3339) + } + apply := &unstructured.Unstructured{Object: map[string]any{ + "metadata": withResourceVersion(map[string]any{"namespace": namespace, "name": runName}, resourceVersion), + "status": fields, + }} + apply.SetGroupVersionKind(v1beta2.DynamoGraphDeploymentRunGVK) + return apply +} + +// normalizeJSON round-trips a value through JSON so it only holds the types unstructured +// objects allow (int64, float64, string, bool, maps, slices). +func normalizeJSON(value map[string]any) (map[string]any, error) { + out := map[string]any{} + if value == nil { + return out, nil + } + raw, err := json.Marshal(value) + if err != nil { + return nil, err + } + if err := json.Unmarshal(raw, &out); err != nil { + return nil, err + } + return out, nil +} diff --git a/deploy/operator/internal/dgdrrunpublisher/writer_envtest_test.go b/deploy/operator/internal/dgdrrunpublisher/writer_envtest_test.go new file mode 100644 index 000000000000..77552d82ebb7 --- /dev/null +++ b/deploy/operator/internal/dgdrrunpublisher/writer_envtest_test.go @@ -0,0 +1,265 @@ +//go:build !clustertest + +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dgdrrunpublisher + +import ( + "context" + "strings" + "testing" + "time" + + v1beta2 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta2" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/dgdcreconcile" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/testing/operatorenv" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// These tests send the publisher's real server-side-apply payloads to an API server that +// has the v1beta2 CRDs installed. The reconciler tests use a fake writer; this is the +// check that the payloads are accepted by the real schema, that the publisher owns what +// it writes, and that the resourceVersion constraints behave as the reconciler assumes. + +type writerFixture struct { + t *testing.T + ctx context.Context + client client.Client + run *v1beta2.DynamoGraphDeploymentRun + writer *clientWriter +} + +func newWriterFixture(t *testing.T) *writerFixture { + t.Helper() + env := operatorenv.New(operatorenv.Options{ + // The publisher has no webhooks; the environment only requires the hook to exist. + SetupWebhooks: func(ctrl.Manager, operatorenv.WebhookSetupOptions) error { return nil }, + }).RunT(t) + ctx := context.Background() + c := env.Client() + + run := &v1beta2.DynamoGraphDeploymentRun{ + ObjectMeta: metav1.ObjectMeta{Namespace: env.Namespace(), Name: "publisher-run"}, + Spec: v1beta2.DynamoGraphDeploymentRunSpec{ + NativeSearchInput: v1beta2.NativeSearchInput{Model: &v1beta2.ModelSpec{Name: "Qwen/Qwen3-0.6B"}}, + }, + } + if err := c.Create(ctx, run); err != nil { + t.Fatalf("create run: %v", err) + } + return &writerFixture{ + t: t, + ctx: ctx, + client: c, + run: run, + writer: &clientWriter{ + Client: c, + Reader: c, + Namespace: run.Namespace, + RunName: run.Name, + RunUID: string(run.UID), + }, + } +} + +func (f *writerFixture) name(id string) string { + return CandidateName(f.run.Name, string(f.run.UID), id) +} + +func (f *writerFixture) candidate(name string) *v1beta2.DynamoGraphDeploymentCandidate { + f.t.Helper() + var got v1beta2.DynamoGraphDeploymentCandidate + if err := f.client.Get(f.ctx, types.NamespacedName{Namespace: f.run.Namespace, Name: name}, &got); err != nil { + f.t.Fatalf("get candidate %s: %v", name, err) + } + return &got +} + +func (f *writerFixture) currentRun() *v1beta2.DynamoGraphDeploymentRun { + f.t.Helper() + var got v1beta2.DynamoGraphDeploymentRun + if err := f.client.Get(f.ctx, client.ObjectKeyFromObject(f.run), &got); err != nil { + f.t.Fatalf("get run: %v", err) + } + return &got +} + +func envtestCandidate() dgdcreconcile.DesiredCandidate { + return dgdcreconcile.DesiredCandidate{ + ID: "a", + Spec: validManifest + "---\napiVersion: v1\nkind: ConfigMap\nmetadata:\n name: companion\n", + Parameters: map[string]any{"tp": 2}, + Metrics: map[string]any{"throughput": 12.5}, + } +} + +func hasApplyManager(fields []metav1.ManagedFieldsEntry, subresource string) bool { + for _, entry := range fields { + if entry.Manager == fieldOwner && entry.Operation == metav1.ManagedFieldsOperationApply && entry.Subresource == subresource { + return true + } + } + return false +} + +func TestEnvtestCandidateApplyIsAcceptedAndIdempotent(t *testing.T) { + f := newWriterFixture(t) + name := f.name("a") + + stored, err := f.writer.CreateCandidate(f.ctx, f.run, name, envtestCandidate()) + if err != nil { + t.Fatalf("create candidate: %v", err) + } + if stored.ResourceVersion == "" || stored.Populated { + t.Fatalf("stored = %+v, want a resourceVersion and no status yet", stored) + } + + got := f.candidate(name) + if got.Labels[LabelRunUID] != string(f.run.UID) || got.Labels[LabelCandidateID] != "a" { + t.Errorf("labels = %v", got.Labels) + } + if owner := metav1.GetControllerOf(got); owner == nil || owner.UID != f.run.UID || owner.Kind != "DynamoGraphDeploymentRun" { + t.Errorf("controller owner = %+v, want the run", owner) + } + if !strings.Contains(got.Annotations[AnnotationAdditionalResources], "kind: ConfigMap") { + t.Errorf("companion annotation = %q", got.Annotations[AnnotationAdditionalResources]) + } + if !hasApplyManager(got.ManagedFields, "") { + t.Errorf("no apply manager %q on the candidate: %+v", fieldOwner, got.ManagedFields) + } + + again, err := f.writer.CreateCandidate(f.ctx, f.run, name, envtestCandidate()) + if err != nil { + t.Fatalf("re-apply candidate: %v", err) + } + if again.ResourceVersion != stored.ResourceVersion { + t.Errorf("re-applying the same candidate changed resourceVersion %s -> %s", stored.ResourceVersion, again.ResourceVersion) + } +} + +func TestEnvtestCandidateStatusIsWrittenOnceAndGuardedByResourceVersion(t *testing.T) { + f := newWriterFixture(t) + name := f.name("a") + candidate := envtestCandidate() + + stored, err := f.writer.CreateCandidate(f.ctx, f.run, name, candidate) + if err != nil { + t.Fatalf("create candidate: %v", err) + } + if err := f.writer.PopulateStatus(f.ctx, name, stored.ResourceVersion, candidate); err != nil { + t.Fatalf("populate status: %v", err) + } + got := f.candidate(name) + if got.Status.Metrics == nil { + t.Fatal("status.metrics is not set after PopulateStatus") + } + if !hasApplyManager(got.ManagedFields, "status") { + t.Errorf("no status apply manager %q: %+v", fieldOwner, got.ManagedFields) + } + + // A decision made from the version before the status was written must conflict. + err = f.writer.PopulateStatus(f.ctx, name, stored.ResourceVersion, candidate) + if !apierrors.IsConflict(err) { + t.Fatalf("populate with a stale resourceVersion: err = %v, want a conflict", err) + } + + // Re-applying the candidate after its status exists reports it as populated. + after, err := f.writer.CreateCandidate(f.ctx, f.run, name, candidate) + if err != nil { + t.Fatalf("re-apply candidate: %v", err) + } + if !after.Populated { + t.Error("a candidate with status was reported as not populated") + } +} + +func TestEnvtestRunStatusIsOwnedByThePublisher(t *testing.T) { + f := newWriterFixture(t) + first := time.Date(2026, 10, 22, 12, 0, 0, 0, time.UTC) + + staleVersion := f.currentRun().ResourceVersion + if err := f.writer.UpdateRunStatus(f.ctx, staleVersion, RunStatus{ + Published: 2, + LastProgress: first, + CandidateNames: []string{f.name("a"), f.name("b")}, + }); err != nil { + t.Fatalf("update run status: %v", err) + } + run := f.currentRun() + if len(run.Status.CandidateRefs) != 2 || run.Status.CandidateRefs[0].Name != f.name("a") { + t.Fatalf("candidateRefs = %+v", run.Status.CandidateRefs) + } + if run.Status.Progress == nil || run.Status.Progress.Candidates == nil || run.Status.Progress.Candidates.Published != 2 { + t.Fatalf("progress = %+v, want 2 published", run.Status.Progress) + } + if run.Status.LastProgressTime == nil || !run.Status.LastProgressTime.UTC().Equal(first) { + t.Fatalf("lastProgressTime = %v, want %v", run.Status.LastProgressTime, first) + } + if !hasApplyManager(run.ManagedFields, "status") { + t.Errorf("no status apply manager %q: %+v", fieldOwner, run.ManagedFields) + } + + // A write based on the version read before the first write conflicts. + err := f.writer.UpdateRunStatus(f.ctx, staleVersion, RunStatus{Published: 1, CandidateNames: []string{f.name("a")}}) + if !apierrors.IsConflict(err) { + t.Fatalf("update with a stale resourceVersion: err = %v, want a conflict", err) + } + + // Fields the publisher owns and omits are removed: this is why the reconciler carries + // the existing progress time into every payload. + if err := f.writer.UpdateRunStatus(f.ctx, run.ResourceVersion, RunStatus{Published: 1, CandidateNames: []string{f.name("a")}}); err != nil { + t.Fatalf("shrinking update: %v", err) + } + run = f.currentRun() + if len(run.Status.CandidateRefs) != 1 || run.Status.Progress.Candidates.Published != 1 { + t.Errorf("status after shrinking = %+v", run.Status) + } + if run.Status.LastProgressTime != nil { + t.Errorf("lastProgressTime = %v, want it removed when the payload omits it", run.Status.LastProgressTime) + } +} + +func TestEnvtestDeleteCandidateHonorsTheUIDPrecondition(t *testing.T) { + f := newWriterFixture(t) + name := f.name("a") + if _, err := f.writer.CreateCandidate(f.ctx, f.run, name, envtestCandidate()); err != nil { + t.Fatalf("create candidate: %v", err) + } + uid := f.candidate(name).UID + + if err := f.writer.DeleteCandidate(f.ctx, name, types.UID("not-the-uid")); err == nil { + t.Fatal("deleting with the wrong UID succeeded; a recreated candidate would be lost") + } + f.candidate(name) // still there + + if err := f.writer.DeleteCandidate(f.ctx, name, uid); err != nil { + t.Fatalf("delete with the right UID: %v", err) + } + var gone v1beta2.DynamoGraphDeploymentCandidate + err := f.client.Get(f.ctx, types.NamespacedName{Namespace: f.run.Namespace, Name: name}, &gone) + if !apierrors.IsNotFound(err) { + t.Fatalf("candidate still exists after delete: %v", err) + } + if err := f.writer.DeleteCandidate(f.ctx, name, uid); err != nil { + t.Fatalf("deleting a missing candidate must not fail: %v", err) + } +} diff --git a/deploy/operator/internal/dynamo/component_common.go b/deploy/operator/internal/dynamo/component_common.go index 404f07a544a9..ca4e1f2e9506 100644 --- a/deploy/operator/internal/dynamo/component_common.go +++ b/deploy/operator/internal/dynamo/component_common.go @@ -10,7 +10,7 @@ import ( "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" commonconsts "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" controller_common "github.com/ai-dynamo/dynamo/deploy/operator/internal/controller_common" - "github.com/ai-dynamo/dynamo/deploy/operator/internal/runtimeversion" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/versionoverride" corev1 "k8s.io/api/core/v1" "k8s.io/utils/ptr" ) @@ -70,7 +70,7 @@ type ComponentContext struct { Infrastructure configv1alpha1.InfrastructureConfiguration EPPConfig *v1beta1.EPPConfig WorkerHashSuffix string - RuntimeVersion *runtimeversion.Version + RuntimeVersion *versionoverride.Version } func (b *BaseComponentDefaults) GetBaseContainer(context ComponentContext) (corev1.Container, error) { diff --git a/deploy/operator/internal/dynamo/component_frontend_test.go b/deploy/operator/internal/dynamo/component_frontend_test.go index ac178e3b8be0..75129dd10f5d 100644 --- a/deploy/operator/internal/dynamo/component_frontend_test.go +++ b/deploy/operator/internal/dynamo/component_frontend_test.go @@ -11,7 +11,7 @@ import ( configv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/config/v1alpha1" "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" commonconsts "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" - "github.com/ai-dynamo/dynamo/deploy/operator/internal/runtimeversion" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/versionoverride" "github.com/stretchr/testify/require" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/util/intstr" @@ -21,13 +21,13 @@ import ( func TestFrontendDefaultsStartupProbeVersionGate(t *testing.T) { tests := []struct { name string - runtimeVersion *runtimeversion.Version + runtimeVersion *versionoverride.Version wantEnabled bool }{ {name: "unknown legacy runtime"}, - {name: "older runtime", runtimeVersion: &runtimeversion.Version{Major: 1, Minor: 5, Patch: 9}}, - {name: "minimum supported runtime", runtimeVersion: &runtimeversion.Version{Major: 1, Minor: 6}, wantEnabled: true}, - {name: "newer runtime", runtimeVersion: &runtimeversion.Version{Major: 2}, wantEnabled: true}, + {name: "older runtime", runtimeVersion: &versionoverride.Version{Major: 1, Minor: 5, Patch: 9}}, + {name: "minimum supported runtime", runtimeVersion: &versionoverride.Version{Major: 1, Minor: 6}, wantEnabled: true}, + {name: "newer runtime", runtimeVersion: &versionoverride.Version{Major: 2}, wantEnabled: true}, } for _, tt := range tests { diff --git a/deploy/operator/internal/dynamo/component_namespace_test.go b/deploy/operator/internal/dynamo/component_namespace_test.go index 5919964c0a86..44aebfefcb3f 100644 --- a/deploy/operator/internal/dynamo/component_namespace_test.go +++ b/deploy/operator/internal/dynamo/component_namespace_test.go @@ -11,7 +11,7 @@ import ( configv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/config/v1alpha1" v1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" commonconsts "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" - "github.com/ai-dynamo/dynamo/deploy/operator/internal/runtimeversion" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/versionoverride" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" corev1 "k8s.io/api/core/v1" @@ -23,13 +23,13 @@ func TestComponentNamespacePrefixRuntimeCompatibility(t *testing.T) { t.Log("render frontend and EPP defaults across the strict-discovery release boundary") versions := []struct { name string - version *runtimeversion.Version + version *versionoverride.Version strict bool }{ {name: "unknown runtime"}, - {name: "older runtime", version: &runtimeversion.Version{Major: 1, Minor: 5, Patch: 9}}, - {name: "supported runtime", version: &runtimeversion.Version{Major: 1, Minor: 6, Patch: 0}, strict: true}, - {name: "newer runtime", version: &runtimeversion.Version{Major: 1, Minor: 7, Patch: 0}, strict: true}, + {name: "older runtime", version: &versionoverride.Version{Major: 1, Minor: 5, Patch: 9}}, + {name: "supported runtime", version: &versionoverride.Version{Major: 1, Minor: 6, Patch: 0}, strict: true}, + {name: "newer runtime", version: &versionoverride.Version{Major: 1, Minor: 7, Patch: 0}, strict: true}, } for _, componentType := range []string{commonconsts.ComponentTypeFrontend, commonconsts.ComponentTypeEPP} { @@ -60,7 +60,7 @@ func TestLegacyEPPDoesNotEnableStrictNamespacePrefix(t *testing.T) { container, err := NewEPPDefaults().GetBaseContainer(ComponentContext{ DynamoNamespace: "default-foo", ComponentType: commonconsts.ComponentTypeEPP, - RuntimeVersion: &runtimeversion.Version{Major: 1, Minor: 6, Patch: 0}, + RuntimeVersion: &versionoverride.Version{Major: 1, Minor: 6, Patch: 0}, EPPConfig: &v1beta1.EPPConfig{}, }) require.NoError(t, err) diff --git a/deploy/operator/internal/dynamo/component_worker_test.go b/deploy/operator/internal/dynamo/component_worker_test.go index aa39447c5b40..2c399d60724e 100644 --- a/deploy/operator/internal/dynamo/component_worker_test.go +++ b/deploy/operator/internal/dynamo/component_worker_test.go @@ -9,14 +9,14 @@ import ( "testing" commonconsts "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" - "github.com/ai-dynamo/dynamo/deploy/operator/internal/runtimeversion" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/versionoverride" "github.com/stretchr/testify/require" ) func TestWorkerDefaultsCanaryHealthCheckVersionGate(t *testing.T) { tests := []struct { name string - runtimeVersion *runtimeversion.Version + runtimeVersion *versionoverride.Version wantEnabled string }{ { @@ -25,17 +25,17 @@ func TestWorkerDefaultsCanaryHealthCheckVersionGate(t *testing.T) { }, { name: "older runtime", - runtimeVersion: &runtimeversion.Version{Major: 1, Minor: 4, Patch: 9}, + runtimeVersion: &versionoverride.Version{Major: 1, Minor: 4, Patch: 9}, wantEnabled: "false", }, { name: "minimum supported runtime", - runtimeVersion: &runtimeversion.Version{Major: 1, Minor: 5, Patch: 0}, + runtimeVersion: &versionoverride.Version{Major: 1, Minor: 5, Patch: 0}, wantEnabled: "true", }, { name: "newer runtime", - runtimeVersion: &runtimeversion.Version{Major: 2, Minor: 0, Patch: 0}, + runtimeVersion: &versionoverride.Version{Major: 2, Minor: 0, Patch: 0}, wantEnabled: "true", }, } @@ -64,7 +64,7 @@ func TestWorkerDefaultsCanaryHealthCheckVersionGate(t *testing.T) { func TestWorkerDefaultsLivenessFailureThresholdVersionGate(t *testing.T) { tests := []struct { name string - runtimeVersion *runtimeversion.Version + runtimeVersion *versionoverride.Version wantFailureCount int32 }{ { @@ -73,17 +73,17 @@ func TestWorkerDefaultsLivenessFailureThresholdVersionGate(t *testing.T) { }, { name: "older runtime", - runtimeVersion: &runtimeversion.Version{Major: 1, Minor: 4, Patch: 9}, + runtimeVersion: &versionoverride.Version{Major: 1, Minor: 4, Patch: 9}, wantFailureCount: 1, }, { name: "minimum supported runtime", - runtimeVersion: &runtimeversion.Version{Major: 1, Minor: 5, Patch: 0}, + runtimeVersion: &versionoverride.Version{Major: 1, Minor: 5, Patch: 0}, wantFailureCount: 3, }, { name: "newer runtime", - runtimeVersion: &runtimeversion.Version{Major: 2, Minor: 0, Patch: 0}, + runtimeVersion: &versionoverride.Version{Major: 2, Minor: 0, Patch: 0}, wantFailureCount: 3, }, } diff --git a/deploy/operator/internal/dynamo/graph.go b/deploy/operator/internal/dynamo/graph.go index ef40a40e98ac..e13bcd3a006a 100644 --- a/deploy/operator/internal/dynamo/graph.go +++ b/deploy/operator/internal/dynamo/graph.go @@ -44,7 +44,7 @@ import ( "github.com/ai-dynamo/dynamo/deploy/operator/internal/features" "github.com/ai-dynamo/dynamo/deploy/operator/internal/features/compatibility" gms "github.com/ai-dynamo/dynamo/deploy/operator/internal/gms" - "github.com/ai-dynamo/dynamo/deploy/operator/internal/runtimeversion" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/versionoverride" grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" "github.com/imdario/mergo" "google.golang.org/protobuf/types/known/wrapperspb" @@ -2176,8 +2176,8 @@ func mergeFrontendSidecarDefaults(podSpec *corev1.PodSpec, sidecarName string, p } // Resolve the frontend image independently of the component runtime. - var resolvedRuntimeVersion *runtimeversion.Version - if version, err := runtimeversion.ParseImageVersion(podSpec.Containers[i].Image); err == nil { + var resolvedRuntimeVersion *versionoverride.Version + if version, err := versionoverride.ParseImageVersion(podSpec.Containers[i].Image); err == nil { resolvedRuntimeVersion = &version } @@ -2270,8 +2270,8 @@ func generateComponentContext(component *v1beta1.DynamoComponentDeploymentShared if runtime := GetDynamoContainer(component); runtime != nil { image = runtime.Image } - var resolvedRuntimeVersion *runtimeversion.Version - if version, err := runtimeversion.Resolve(image, component.RuntimeVersionOverride); err == nil { + var resolvedRuntimeVersion *versionoverride.Version + if version, err := versionoverride.Resolve(image, component.RuntimeVersionOverride); err == nil { resolvedRuntimeVersion = &version } else if component.RuntimeVersionOverride != "" { return ComponentContext{}, fmt.Errorf("resolve runtime version override: %w", err) diff --git a/deploy/operator/internal/dynamo/hash.go b/deploy/operator/internal/dynamo/hash.go index 4dd87b283c00..de2be302a719 100644 --- a/deploy/operator/internal/dynamo/hash.go +++ b/deploy/operator/internal/dynamo/hash.go @@ -26,7 +26,7 @@ import ( "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" "github.com/ai-dynamo/dynamo/deploy/operator/internal/provideroverride" - "github.com/ai-dynamo/dynamo/deploy/operator/internal/runtimeversion" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/versionoverride" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" "k8s.io/utils/ptr" ) @@ -34,7 +34,7 @@ import ( const dgdWorkerHashPlaceholderValue = "worker-hash-placeholder" // Preserve hashes for runtimes released before version hashing. -var minimumHashedRuntimeVersion = runtimeversion.Version{Major: 1, Minor: 5, Patch: 0} +var minimumHashedRuntimeVersion = versionoverride.Version{Major: 1, Minor: 5, Patch: 0} // ComputeDGDWorkersSpecHash computes the v2 worker hash from the worker DCDs // generated by the same DGD-to-DCD path used by reconciliation. The active @@ -223,7 +223,7 @@ func resolvedRuntimeVersionForHash(component *v1beta1.DynamoComponentDeploymentS if runtime := GetDynamoContainer(component); runtime != nil { image = runtime.Image } - version, err := runtimeversion.Resolve(image, component.RuntimeVersionOverride) + version, err := versionoverride.Resolve(image, component.RuntimeVersionOverride) if err != nil || version.Compare(minimumHashedRuntimeVersion) < 0 { return "" } diff --git a/deploy/operator/internal/features/runtime/gate.go b/deploy/operator/internal/features/runtime/gate.go index 805099b09e95..66a3aedab213 100644 --- a/deploy/operator/internal/features/runtime/gate.go +++ b/deploy/operator/internal/features/runtime/gate.go @@ -5,16 +5,16 @@ package runtime -import "github.com/ai-dynamo/dynamo/deploy/operator/internal/runtimeversion" +import "github.com/ai-dynamo/dynamo/deploy/operator/internal/versionoverride" // Gate controls runtime-version-dependent operator behavior. type Gate struct { Name string - MinRuntimeVersion runtimeversion.Version + MinRuntimeVersion versionoverride.Version } // Enabled reports whether a known runtime version meets the feature threshold. -func (g Gate) Enabled(version *runtimeversion.Version) bool { +func (g Gate) Enabled(version *versionoverride.Version) bool { if version == nil { return false } diff --git a/deploy/operator/internal/features/runtime/gate_test.go b/deploy/operator/internal/features/runtime/gate_test.go index c3919a34b988..b9ea9db53738 100644 --- a/deploy/operator/internal/features/runtime/gate_test.go +++ b/deploy/operator/internal/features/runtime/gate_test.go @@ -8,20 +8,20 @@ package runtime import ( "testing" - "github.com/ai-dynamo/dynamo/deploy/operator/internal/runtimeversion" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/versionoverride" ) func TestGateEnabled(t *testing.T) { t.Log("define a feature introduced by Dynamo runtime 1.4.0") gate := Gate{ Name: "TestFeature", - MinRuntimeVersion: runtimeversion.Version{Major: 1, Minor: 4, Patch: 0}, + MinRuntimeVersion: versionoverride.Version{Major: 1, Minor: 4, Patch: 0}, } t.Log("define versions below, at, and above the feature threshold") tests := []struct { name string - version *runtimeversion.Version + version *versionoverride.Version want bool }{ { @@ -29,16 +29,16 @@ func TestGateEnabled(t *testing.T) { }, { name: "older runtime", - version: &runtimeversion.Version{Major: 1, Minor: 3, Patch: 9}, + version: &versionoverride.Version{Major: 1, Minor: 3, Patch: 9}, }, { name: "minimum supported runtime", - version: &runtimeversion.Version{Major: 1, Minor: 4, Patch: 0}, + version: &versionoverride.Version{Major: 1, Minor: 4, Patch: 0}, want: true, }, { name: "newer runtime", - version: &runtimeversion.Version{Major: 2, Minor: 0, Patch: 0}, + version: &versionoverride.Version{Major: 2, Minor: 0, Patch: 0}, want: true, }, } diff --git a/deploy/operator/internal/features/runtime/gates.go b/deploy/operator/internal/features/runtime/gates.go index b77533226721..ee21c27072c2 100644 --- a/deploy/operator/internal/features/runtime/gates.go +++ b/deploy/operator/internal/features/runtime/gates.go @@ -5,19 +5,19 @@ package runtime -import "github.com/ai-dynamo/dynamo/deploy/operator/internal/runtimeversion" +import "github.com/ai-dynamo/dynamo/deploy/operator/internal/versionoverride" var ( // StrictNamespacePrefix gates operator-managed discovery isolation. StrictNamespacePrefix = Gate{ Name: "StrictNamespacePrefix", - MinRuntimeVersion: runtimeversion.Version{Major: 1, Minor: 6, Patch: 0}, + MinRuntimeVersion: versionoverride.Version{Major: 1, Minor: 6, Patch: 0}, } // FrontendStartupProbe allows the sequential NATS and etcd startup connection windows. FrontendStartupProbe = Gate{ Name: "FrontendStartupProbe", - MinRuntimeVersion: runtimeversion.Version{Major: 1, Minor: 6, Patch: 0}, + MinRuntimeVersion: versionoverride.Version{Major: 1, Minor: 6, Patch: 0}, } // CanaryHealthChecks gates the canary health-check rendering defaults. @@ -26,7 +26,7 @@ var ( // change an existing worker generation. CanaryHealthChecks = Gate{ Name: "CanaryHealthChecks", - MinRuntimeVersion: runtimeversion.Version{Major: 1, Minor: 5, Patch: 0}, + MinRuntimeVersion: versionoverride.Version{Major: 1, Minor: 5, Patch: 0}, } // IncreasedWorkerFailureThreshold gates the worker liveness default. @@ -35,7 +35,7 @@ var ( // change an existing worker generation. IncreasedWorkerFailureThreshold = Gate{ Name: "IncreasedWorkerFailureThreshold", - MinRuntimeVersion: runtimeversion.Version{Major: 1, Minor: 5, Patch: 0}, + MinRuntimeVersion: versionoverride.Version{Major: 1, Minor: 5, Patch: 0}, } // NativeRustEPP gates the native Rust EPP requirement: runtime versions at @@ -44,7 +44,7 @@ var ( // 1.5.0, the first release shipping the native Rust EPP image. NativeRustEPP = Gate{ Name: "NativeRustEPP", - MinRuntimeVersion: runtimeversion.Version{Major: 1, Minor: 5, Patch: 0}, + MinRuntimeVersion: versionoverride.Version{Major: 1, Minor: 5, Patch: 0}, } // PlannerDGDComponentStatus gates Planner discovery of model, runtime, @@ -54,6 +54,6 @@ var ( // PodTemplates. PlannerDGDComponentStatus = Gate{ Name: "PlannerDGDComponentStatus", - MinRuntimeVersion: runtimeversion.Version{Major: 1, Minor: 6, Patch: 0}, + MinRuntimeVersion: versionoverride.Version{Major: 1, Minor: 6, Patch: 0}, } ) diff --git a/deploy/operator/internal/gpu/dcgm_phase_context_test.go b/deploy/operator/internal/gpu/dcgm_phase_context_test.go new file mode 100644 index 000000000000..ba4a86fe6fb6 --- /dev/null +++ b/deploy/operator/internal/gpu/dcgm_phase_context_test.go @@ -0,0 +1,41 @@ +package gpu + +import ( + "context" + "testing" + "time" +) + +func TestDCGMPhaseContextReservesTimeForTheFallback(t *testing.T) { + outer, cancelOuter := context.WithTimeout(t.Context(), 4*time.Second) + defer cancelOuter() + outerDeadline, _ := outer.Deadline() + + t.Log("With the fallback enabled the DCGM phase ends well before the outer deadline") + phase, cancel := dcgmPhaseContext(outer, func() bool { return true }) + defer cancel() + phaseDeadline, ok := phase.Deadline() + if !ok { + t.Fatal("phase context has no deadline") + } + if remaining := outerDeadline.Sub(phaseDeadline); remaining < time.Second { + t.Fatalf("only %v is left for the fallback, want at least 1s", remaining) + } + + t.Log("Without the fallback the whole context is used") + same, cancelSame := dcgmPhaseContext(outer, func() bool { return false }) + defer cancelSame() + if same != outer { + t.Fatal("context was narrowed although there is no fallback to reserve time for") + } + + t.Log("A context without a deadline is left alone") + bare, cancelBare := dcgmPhaseContext(t.Context(), func() bool { + t.Fatal("the fallback callback was asked although the context has no deadline") + return true + }) + defer cancelBare() + if _, hasDeadline := bare.Deadline(); hasDeadline { + t.Fatal("a deadline was invented for a context that had none") + } +} diff --git a/deploy/operator/internal/gpu/hardware_discoverer.go b/deploy/operator/internal/gpu/hardware_discoverer.go new file mode 100644 index 000000000000..988ffb428d72 --- /dev/null +++ b/deploy/operator/internal/gpu/hardware_discoverer.go @@ -0,0 +1,162 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package gpu + +import ( + "context" + "errors" + "fmt" + "strings" + "time" + + nvidiacomv1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/log" +) + +// Discoverer finds the cluster's GPUs the way the DGDR controller does: DCGM first, then +// node labels when node-label discovery is enabled. It is a plain value so callers other +// than the v1beta1 reconciler (the v1beta2 admission webhook) share the same order and the +// same error wording. +type Discoverer struct { + Reader client.Reader + // DCGM scrapes the DCGM exporter. Nil skips it. + DCGM *GPUDiscovery + Cache *GPUDiscoveryCache + // NodeLabels reports whether the node-label fallback is enabled (the GPUDiscovery feature + // gate). It is a func so the gate is only read when DCGM did not answer, as the v1beta1 + // reconciler always did. Nil means disabled. + NodeLabels func() bool +} + +// Discover returns the discovered GPUs, restricted to filterSKU when it is set. An error +// means no backend produced a result. +func (d *Discoverer) Discover(ctx context.Context, filterSKU nvidiacomv1beta1.GPUSKUType) (*GPUInfo, error) { + logger := log.FromContext(ctx) + if d == nil || d.Reader == nil { + return nil, errors.New("auto-discovery failed: APIReader is not configured") + } + + var dcgmErr error + if d.DCGM != nil { + // A slow exporter must not leave the node-label fallback with an expired context. + dcgmCtx, cancel := dcgmPhaseContext(ctx, d.NodeLabels) + info, err := d.DCGM.DiscoverGPUsFromDCGMFiltered(dcgmCtx, d.Reader, d.Cache, filterSKU) + cancel() + if err == nil { + return info, nil + } + dcgmErr = err + logger.Info("DCGM discovery failed, falling back to node-label discovery", + "reason", DiscoveryFailureReason(err), "error", err.Error()) + } + + if d.NodeLabels == nil || !d.NodeLabels() { + if dcgmErr != nil { + return nil, fmt.Errorf("auto-discovery failed: %w", dcgmErr) + } + return nil, errors.New("auto-discovery failed: node-label discovery is disabled") + } + + info, err := DiscoverGPUsFiltered(ctx, d.Reader, filterSKU) + if err != nil { + logger.Info("Node-label discovery also failed", "error", err.Error()) + return nil, fmt.Errorf("auto-discovery failed: %w", err) + } + return info, nil +} + +// KnownSKU returns the hardware catalog name of the discovered GPU, or empty when the model +// is not one the catalog knows. Unlike the v1beta1 reconciler it never falls back to the raw +// model string: a v1beta2 SKU is an AISimulate catalog name, and an unknown name would only +// fail later inside the search. +func KnownSKU(info *GPUInfo) nvidiacomv1beta1.GPUSKUType { + if info == nil { + return "" + } + if info.System != "" { + return info.System + } + return InferHardwareSystem(info.Model) +} + +// DiscoveryFailureReason classifies a GPU discovery error and +// returns a stable, actionable reason string suitable for structured logging. +// +// The classification is based on known error message patterns produced during: +// - DCGM exporter pod discovery +// - Helm-based GPU operator and DCGM discovery +// - Metrics scraping +// - Prometheus parsing +// +// If the error does not match any known category, "unknown" is returned. +func DiscoveryFailureReason(err error) string { + const unknownReason = "unknown" + + if err == nil { + return unknownReason + } + errMsg := strings.ToLower(err.Error()) + + switch { + case strings.Contains(errMsg, "list pods"): + return "failed to list DCGM exporter pods (RBAC/cluster connectivity issue)" + case strings.Contains(errMsg, "gpu operator is not installed"): + return "GPU Operator not installed in expected namespace" + case strings.Contains(errMsg, "helm init failed"): + return "failed to initialize Helm client (RBAC, kubeconfig, or Helm driver issue)" + case strings.Contains(errMsg, "timeout waiting for dcgm exporter pods"): + return "timeout while waiting for DCGM exporter pods to become ready" + case strings.Contains(errMsg, "http get"): + return "failed to reach DCGM metrics endpoint on pod (network/port issue)" + case strings.Contains(errMsg, "metrics endpoint") && + strings.Contains(errMsg, "status"): + return "DCGM pod metrics endpoint returned non-200 status" + case strings.Contains(errMsg, "parse prometheus metrics"): + return "failed to parse dcgm Prometheus metrics (invalid format)" + case strings.Contains(errMsg, "no gpus detected"): + return "no GPUs detected in dcgm metrics (GPU model or metrics missing)" + case strings.Contains(errMsg, "dcgm is not enabled in the gpu operator"): + return "DCGM is not enabled in the GPU Operator (check GPU Operator configuration and permissions)" + case strings.Contains(errMsg, "failed to scrape any dcgm exporter pod"): + return "failed to scrape any dcgm exporter pod (check DCGM exporter pod status and network connectivity)" + case strings.Contains(errMsg, "no gpu metrics could be parsed from any dcgm pod"): + return "no GPU metrics could be parsed from any DCGM pod (check DCGM exporter pod status and network connectivity)" + case strings.Contains(errMsg, "failed to create helm path"): + return "failed to initialize Helm client (RBAC, kubeconfig, or Helm driver issue)" + } + return unknownReason +} + +// dcgmPhaseContext bounds the DCGM phase of discovery. When ctx has a deadline and fallbackEnabled +// reports that the node-label fallback is enabled, the DCGM phase gets half of what remains, so a slow exporter cannot +// consume the whole admission budget and leave the fallback with an expired context. Otherwise +// ctx is returned unchanged. The returned cancel function must be called. +func dcgmPhaseContext(ctx context.Context, fallbackEnabled func() bool) (context.Context, context.CancelFunc) { + deadline, ok := ctx.Deadline() + // fallbackEnabled is only asked when there is a deadline to share: without one nothing is reserved, + // and the callback may depend on state that a caller without a deadline never configured. + if !ok || fallbackEnabled == nil || !fallbackEnabled() { + return ctx, func() {} + } + remaining := time.Until(deadline) + if remaining <= 0 { + return ctx, func() {} + } + return context.WithTimeout(ctx, remaining/2) +} diff --git a/deploy/operator/internal/gpu/hardware_discoverer_test.go b/deploy/operator/internal/gpu/hardware_discoverer_test.go new file mode 100644 index 000000000000..ef0493c42197 --- /dev/null +++ b/deploy/operator/internal/gpu/hardware_discoverer_test.go @@ -0,0 +1,90 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package gpu + +import ( + "context" + "errors" + "strings" + "testing" + + nvidiacomv1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" + "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +func TestDiscovererWithoutReader(t *testing.T) { + var nilDiscoverer *Discoverer + for _, d := range []*Discoverer{nilDiscoverer, {}} { + if _, err := d.Discover(context.Background(), ""); err == nil || !strings.Contains(err.Error(), "APIReader is not configured") { + t.Fatalf("got %v", err) + } + } +} + +func TestDiscovererWithNoBackend(t *testing.T) { + d := &Discoverer{Reader: fake.NewClientBuilder().Build()} + if _, err := d.Discover(context.Background(), ""); err == nil || !strings.Contains(err.Error(), "node-label discovery is disabled") { + t.Fatalf("got %v", err) + } +} + +func TestDiscovererNodeLabelsFindNothing(t *testing.T) { + d := &Discoverer{Reader: fake.NewClientBuilder().Build(), NodeLabels: func() bool { return true }} + if _, err := d.Discover(context.Background(), ""); err == nil || !strings.Contains(err.Error(), "auto-discovery failed") { + t.Fatalf("an empty cluster must be an error, got %v", err) + } +} + +func TestKnownSKU(t *testing.T) { + cases := []struct { + name string + info *GPUInfo + want nvidiacomv1beta1.GPUSKUType + }{ + {"nil", nil, ""}, + {"discovered system wins", &GPUInfo{System: "h100_sxm", Model: "NVIDIA H200"}, "h100_sxm"}, + {"inferred from the model", &GPUInfo{Model: "NVIDIA H200"}, "h200_sxm"}, + {"unknown model is not passed through", &GPUInfo{Model: "NVIDIA RTX 6000 Ada"}, ""}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := KnownSKU(tc.info); got != tc.want { + t.Fatalf("got %q want %q", got, tc.want) + } + }) + } +} + +func TestDiscoveryFailureReason(t *testing.T) { + cases := map[string]string{ + "list pods: forbidden": "failed to list DCGM exporter pods (RBAC/cluster connectivity issue)", + "GPU Operator is not installed": "GPU Operator not installed in expected namespace", + "http get failed": "failed to reach DCGM metrics endpoint on pod (network/port issue)", + "metrics endpoint returned status": "DCGM pod metrics endpoint returned non-200 status", + "no GPUs detected": "no GPUs detected in dcgm metrics (GPU model or metrics missing)", + "something else": "unknown", + } + for msg, want := range cases { + if got := DiscoveryFailureReason(errors.New(msg)); got != want { + t.Errorf("%q: got %q, want %q", msg, got, want) + } + } + if got := DiscoveryFailureReason(nil); got != "unknown" { + t.Errorf("nil: got %q", got) + } +} diff --git a/deploy/operator/internal/runtimeversion/runtimeversion.go b/deploy/operator/internal/versionoverride/versionoverride.go similarity index 99% rename from deploy/operator/internal/runtimeversion/runtimeversion.go rename to deploy/operator/internal/versionoverride/versionoverride.go index eab91c6c48d8..c400bce11af7 100644 --- a/deploy/operator/internal/runtimeversion/runtimeversion.go +++ b/deploy/operator/internal/versionoverride/versionoverride.go @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package runtimeversion +package versionoverride import ( "cmp" diff --git a/deploy/operator/internal/runtimeversion/runtimeversion_test.go b/deploy/operator/internal/versionoverride/versionoverride_test.go similarity index 99% rename from deploy/operator/internal/runtimeversion/runtimeversion_test.go rename to deploy/operator/internal/versionoverride/versionoverride_test.go index edc26679c343..8cccb8005eef 100644 --- a/deploy/operator/internal/runtimeversion/runtimeversion_test.go +++ b/deploy/operator/internal/versionoverride/versionoverride_test.go @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package runtimeversion +package versionoverride import "testing" diff --git a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_handler.go b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_handler.go index a828ad8f145b..f88a48fb6c46 100644 --- a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_handler.go +++ b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_handler.go @@ -63,6 +63,9 @@ type DGDRDefaulter struct { // dynamo-planner: tag does not exist (pre-release charts: // rc, nightly); empty keeps the derived default. DefaultImage string + // v1beta2 configures the defaults of Requests submitted as v1beta2, which reach this + // handler as a v1beta1 object carrying the spec.v1beta2 envelope. + v1beta2 DGDRV1beta2Options } // NewDGDRDefaulter creates a new DGDRDefaulter with the given operator version @@ -71,6 +74,13 @@ func NewDGDRDefaulter(operatorVersion, defaultImage string) *DGDRDefaulter { return &DGDRDefaulter{OperatorVersion: operatorVersion, DefaultImage: defaultImage} } +// WithV1beta2 sets the defaults of v1beta2 Requests and returns the defaulter. +func (d *DGDRDefaulter) WithV1beta2(options DGDRV1beta2Options) *DGDRDefaulter { + options.AISimulateImage = defaultPlannerImageFor(d.OperatorVersion, options.AISimulateImage) + d.v1beta2 = options + return d +} + // Default implements admission.CustomDefaulter. // Native API defaults apply on CREATE and UPDATE. If spec.image is not set on // CREATE, derives a default image from the operator version. @@ -88,11 +98,23 @@ func (d *DGDRDefaulter) Default(ctx context.Context, obj runtime.Object) error { req, err := admission.RequestFromContext(ctx) if err != nil { - logger.Error(err, "failed to get admission request from context, skipping defaulting") - return nil + return fmt.Errorf("get admission request: %w", err) } - - if dgdr.Spec.V1Beta2 != nil { + requestVersion := internalwebhook.RequestedVersion(req) + + // The original request version decides what spec.v1beta2 means. A native v1beta2 Request + // is defaulted from its carrier on CREATE and UPDATE; only CREATE + // and an UPDATE that changes spec.hardware read the cluster. + if requestVersion == internalwebhook.VersionV1Beta2 && dgdr.Spec.V1Beta2 != nil { + oldSpec, err := decodeOldSpecV1beta2(req.Operation, req.OldObject.Raw) + if err != nil { + return err + } + return defaultDynamoGraphDeploymentRequestV1beta2(ctx, dgdr, req.Operation, oldSpec, d.v1beta2) + } + // A foreign-version view (a carrier written or updated through the wrong API version, or + // a v1beta1 payload written through v1beta2) is read-only; validation enforces that. + if requestVersion == internalwebhook.VersionV1Beta2 || dgdr.Spec.V1Beta2 != nil { return nil } @@ -111,7 +133,7 @@ func (d *DGDRDefaulter) Default(ctx context.Context, obj runtime.Object) error { // Image derivation remains create-only so updates do not rewrite an // intentionally cleared image. if req.Operation == admissionv1.Create && dgdr.Spec.Image == "" { - if img := d.defaultImageFor(); img != "" { + if img := defaultPlannerImageFor(d.OperatorVersion, d.DefaultImage); img != "" { dgdr.Spec.Image = img logger.Info("defaulted spec.image from operator version", "name", dgdr.Name, @@ -124,14 +146,13 @@ func (d *DGDRDefaulter) Default(ctx context.Context, obj runtime.Object) error { return nil } -// defaultImageFor returns the profiler image for spec.image: DefaultImage when -// set, else the derived image with a canonical semver tag, or an empty string -// when the operator version cannot be parsed. -func (d *DGDRDefaulter) defaultImageFor() string { - if d.DefaultImage != "" { - return d.DefaultImage +// defaultPlannerImageFor returns explicitImage when set, else the planner image with a +// canonical operator-version tag, or an empty string when the version cannot be parsed. +func defaultPlannerImageFor(operatorVersion, explicitImage string) string { + if explicitImage != "" { + return explicitImage } - version, err := semver.NewVersion(d.OperatorVersion) + version, err := semver.NewVersion(operatorVersion) if err != nil { return "" } diff --git a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_handler_test.go b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_handler_test.go index 4f2c35382611..48fafb9ead7c 100644 --- a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_handler_test.go +++ b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_handler_test.go @@ -28,7 +28,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/webhook/admission" ) -func TestDGDRDefaulter_defaultImageFor(t *testing.T) { +func TestDefaultPlannerImageFor(t *testing.T) { // Note: the default planner image is only published starting from Dynamo 1.1.0, // so these tests use 1.1.0 as the earliest known valid version. tests := []struct { @@ -77,10 +77,9 @@ func TestDGDRDefaulter_defaultImageFor(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - d := NewDGDRDefaulter(tt.operatorVersion, tt.defaultImage) - got := d.defaultImageFor() + got := defaultPlannerImageFor(tt.operatorVersion, tt.defaultImage) if got != tt.expectedImage { - t.Errorf("defaultImageFor() = %q, want %q", got, tt.expectedImage) + t.Errorf("defaultPlannerImageFor() = %q, want %q", got, tt.expectedImage) } }) } diff --git a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_routing_test.go b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_routing_test.go new file mode 100644 index 000000000000..416fe004f280 --- /dev/null +++ b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_routing_test.go @@ -0,0 +1,146 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package defaulting + +import ( + "bytes" + "context" + "encoding/json" + "testing" + + nvidiacomv1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" + admissionv1 "k8s.io/api/admission/v1" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" +) + +// ctxForRequest builds the admission context of a request the API server converted to +// v1beta1. requestVersion is the version the client used; empty means no conversion. +func ctxForRequest(op admissionv1.Operation, requestVersion string) context.Context { + gvk := nvidiacomv1beta1.DynamoGraphDeploymentRequestGVK + req := admission.Request{AdmissionRequest: admissionv1.AdmissionRequest{ + Operation: op, + Kind: metav1.GroupVersionKind{Group: gvk.Group, Version: gvk.Version, Kind: gvk.Kind}, + }} + if requestVersion != "" { + req.RequestKind = &metav1.GroupVersionKind{Group: gvk.Group, Version: requestVersion, Kind: gvk.Kind} + } + return admission.NewContextWithRequest(context.Background(), req) +} + +const modelOnlyCarrier = `{"model":{"name":"Qwen/Qwen3-32B"}}` + +func carrierRequest() *nvidiacomv1beta1.DynamoGraphDeploymentRequest { + return &nvidiacomv1beta1.DynamoGraphDeploymentRequest{ + ObjectMeta: metav1.ObjectMeta{Name: "r", Namespace: "default"}, + Spec: nvidiacomv1beta1.DynamoGraphDeploymentRequestSpec{ + V1Beta2: &apiextensionsv1.JSON{Raw: []byte(modelOnlyCarrier)}, + }, + } +} + +func aisimulateImageOf(t *testing.T, raw []byte) string { + t.Helper() + var spec map[string]any + if err := json.Unmarshal(raw, &spec); err != nil { + t.Fatalf("decode carrier: %v", err) + } + aisimulate, _ := spec["aisimulate"].(map[string]any) + image, _ := aisimulate["image"].(string) + return image +} + +func TestDGDRDefaulter_RoutesByRequestedVersion(t *testing.T) { + for _, tt := range []struct { + name string + options DGDRV1beta2Options + want string + }{ + { + name: "an omitted image is derived from the operator version", + want: "nvcr.io/nvidia/ai-dynamo/dynamo-planner:1.6.0", + }, + { + name: "an explicit default image overrides derivation", + options: DGDRV1beta2Options{AISimulateImage: "registry.example/aisimulate:custom"}, + want: "registry.example/aisimulate:custom", + }, + } { + t.Run(tt.name, func(t *testing.T) { + d := NewDGDRDefaulter("1.6.0", "").WithV1beta2(tt.options) + dgdr := carrierRequest() + if err := d.Default(ctxForRequest(admissionv1.Create, "v1beta2"), dgdr); err != nil { + t.Fatalf("Default() error = %v", err) + } + if got := aisimulateImageOf(t, dgdr.Spec.V1Beta2.Raw); got != tt.want { + t.Fatalf("aisimulate.image = %q, want %q", got, tt.want) + } + }) + } + + d := NewDGDRDefaulter("1.6.0", "").WithV1beta2(DGDRV1beta2Options{}) + + t.Run("RequestKind decides, not the converted Kind", func(t *testing.T) { + dgdr := carrierRequest() + if err := d.Default(ctxForRequest(admissionv1.Create, "v1beta2"), dgdr); err != nil { + t.Fatalf("Default() error = %v", err) + } + if dgdr.Spec.Image != "" { + t.Fatalf("v1beta1 image defaulting touched a v1beta2 request: %q", dgdr.Spec.Image) + } + }) + + for _, version := range []string{"", "v1beta1", "v1alpha1"} { + t.Run("a foreign carrier is never mutated, requested as "+version, func(t *testing.T) { + for _, op := range []admissionv1.Operation{admissionv1.Create, admissionv1.Update} { + dgdr := carrierRequest() + before := append([]byte(nil), dgdr.Spec.V1Beta2.Raw...) + if err := d.Default(ctxForRequest(op, version), dgdr); err != nil { + t.Fatalf("Default() error = %v", err) + } + if !bytes.Equal(dgdr.Spec.V1Beta2.Raw, before) { + t.Fatalf("%s: carrier changed from %s to %s", op, before, dgdr.Spec.V1Beta2.Raw) + } + if dgdr.Spec.Image != "" { + t.Fatalf("%s: native v1beta1 defaults reached a foreign carrier", op) + } + } + }) + } + + t.Run("a v1beta1 payload written through v1beta2 is left for validation", func(t *testing.T) { + dgdr := &nvidiacomv1beta1.DynamoGraphDeploymentRequest{Spec: nvidiacomv1beta1.DynamoGraphDeploymentRequestSpec{Model: "m"}} + if err := d.Default(ctxForRequest(admissionv1.Create, "v1beta2"), dgdr); err != nil { + t.Fatalf("Default() error = %v", err) + } + if dgdr.Spec.Image != "" { + t.Fatalf("a foreign v1beta1 payload was defaulted: image %q", dgdr.Spec.Image) + } + }) + + t.Run("a native v1beta1 request keeps its defaults", func(t *testing.T) { + dgdr := &nvidiacomv1beta1.DynamoGraphDeploymentRequest{Spec: nvidiacomv1beta1.DynamoGraphDeploymentRequestSpec{Model: "m"}} + if err := d.Default(ctxForRequest(admissionv1.Create, ""), dgdr); err != nil { + t.Fatalf("Default() error = %v", err) + } + if dgdr.Spec.Image == "" { + t.Fatal("native v1beta1 image defaulting stopped working") + } + }) +} diff --git a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta1_options.go b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta1_options.go new file mode 100644 index 000000000000..f5140ee376a0 --- /dev/null +++ b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta1_options.go @@ -0,0 +1,13 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package defaulting + +// DGDRV1beta1Options configures the defaults of v1beta1 Requests. +type DGDRV1beta1Options struct { + // DefaultImage is the profiler image put into spec.image when a Request is created + // without one; empty derives dynamo-planner:. + DefaultImage string +} diff --git a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2.go b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2.go new file mode 100644 index 000000000000..ce134ec09540 --- /dev/null +++ b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2.go @@ -0,0 +1,250 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package defaulting + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + + nvidiacomv1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" + nvidiacomv1beta2 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta2" + admissionv1 "k8s.io/api/admission/v1" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + "k8s.io/utils/ptr" +) + +// defaultMetricV1beta2 is the scalar target an Optimize objective receives when it names none. +const defaultMetricV1beta2 = nvidiacomv1beta2.ObjectiveMetric("Throughput") + +// Defaults of the objects admission creates. Admission is their only owner: the API server +// defaults a mutated object in the webhook's version (v1beta1, whose v1beta2 carrier is +// opaque), so the v1beta2 schema carries no default markers for them. +const ( + defaultStaticISLV1beta2 int32 = 1024 + defaultStaticOSLV1beta2 int32 = 128 + defaultStaticConcurrencyV1beta2 int32 = 10 + defaultMaxCandidatesV1beta2 int32 = 1 +) + +// This file defaults the v1beta2 shape of a DynamoGraphDeploymentRequest. It is separate +// from the canonical defaulting because the v1beta1 hub carries a v1beta2 Request as an +// opaque spec.v1beta2 envelope: conversion cannot apply field-level defaults to it, so the +// handler decodes the envelope, runs this traversal and encodes the result. Names carry the +// API version because the canonical names belong to v1beta1. +// +// Admission owns every v1beta2 default, and each default has this one owner: backends, +// objective.mode, the scalar metric, recommendation.maxCandidates, workload.static, the +// cluster hardware and the AISimulate image. workload.static: {} opts out of the static +// workload. objective.sla has no default: an omitted SLA stays absent. +// spec.aisimulate.parameters never changes a default. It overrides the defaulted typed +// values later, when the input is lowered. + +// defaultDynamoGraphDeploymentRequestV1beta2 defaults the v1beta2 spec that dgdr carries in +// spec.v1beta2. The environment is read once, before any mutation, and only for a CREATE or +// an UPDATE that changes spec.hardware (see resolveHardware). oldSpec is the stored spec on +// UPDATE and nil otherwise. The envelope is rewritten +// only when defaulting changed the spec, so a defaulted object stays byte-identical across +// repeated admissions. +func defaultDynamoGraphDeploymentRequestV1beta2( + ctx context.Context, + dgdr *nvidiacomv1beta1.DynamoGraphDeploymentRequest, + operation admissionv1.Operation, + oldSpec *nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec, + options DGDRV1beta2Options, +) error { + var spec nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec + if err := json.Unmarshal(dgdr.Spec.V1Beta2.Raw, &spec); err != nil { + return fmt.Errorf("decode spec.v1beta2: %w", err) + } + before, err := json.Marshal(&spec) + if err != nil { + return fmt.Errorf("encode spec.v1beta2: %w", err) + } + + defaultDynamoGraphDeploymentRequestSpecV1beta2(&spec, options.resolveDefaults(ctx, operation, oldSpec, &spec)) + + after, err := json.Marshal(&spec) + if err != nil { + return fmt.Errorf("encode spec.v1beta2: %w", err) + } + if !bytes.Equal(before, after) { + dgdr.Spec.V1Beta2 = &apiextensionsv1.JSON{Raw: after} + } + return nil +} + +// decodeOldSpecV1beta2 returns the stored v1beta2 spec an UPDATE replaces. The API server +// converts the old object to the same v1beta1 carrier view as the new one, so the spec sits +// in the same spec.v1beta2 envelope. It returns nil when there is nothing to compare with. +func decodeOldSpecV1beta2( + operation admissionv1.Operation, + oldObject []byte, +) (*nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec, error) { + if operation != admissionv1.Update || len(oldObject) == 0 { + return nil, nil + } + var old nvidiacomv1beta1.DynamoGraphDeploymentRequest + if err := json.Unmarshal(oldObject, &old); err != nil { + return nil, fmt.Errorf("decode old object: %w", err) + } + if old.Spec.V1Beta2 == nil { + return nil, nil + } + var spec nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec + if err := json.Unmarshal(old.Spec.V1Beta2.Raw, &spec); err != nil { + return nil, fmt.Errorf("decode old spec.v1beta2: %w", err) + } + return &spec, nil +} + +// dynamoGraphDeploymentRequestV1beta2DefaultOptions carries the request-scoped values the +// traversal needs. An empty field means the environment had nothing to offer. +type dynamoGraphDeploymentRequestV1beta2DefaultOptions struct { + // hardware is the GPU identity and budget discovered from the cluster. + hardware hardwareDefaultsV1beta2 + // aisimulateImage is the operator's resolved AISimulate image. + aisimulateImage string +} + +// hardwareDefaultsV1beta2 is the hardware the cluster can offer for the values a Request omits. +type hardwareDefaultsV1beta2 struct { + sku string + budget int32 +} + +// defaultDynamoGraphDeploymentRequestSpecV1beta2 defaults spec in place. It applies on +// CREATE and UPDATE and is idempotent; opts carries hardware only when the environment was +// asked (see resolveHardware). spec must not be nil. Explicit values, including empty +// objects, are kept. The functions below mirror the Go API tree and visit children in +// declaration order. +func defaultDynamoGraphDeploymentRequestSpecV1beta2( + spec *nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec, + opts dynamoGraphDeploymentRequestV1beta2DefaultOptions, +) { + defaultDynamoGraphDeploymentRequestNativeSpecV1beta2(&spec.DynamoGraphDeploymentRequestNativeSpec, opts) +} + +func defaultDynamoGraphDeploymentRequestNativeSpecV1beta2( + native *nvidiacomv1beta2.DynamoGraphDeploymentRequestNativeSpec, + opts dynamoGraphDeploymentRequestV1beta2DefaultOptions, +) { + defaultNativeSearchInputV1beta2(&native.NativeSearchInput, opts) +} + +func defaultNativeSearchInputV1beta2( + input *nvidiacomv1beta2.NativeSearchInput, + opts dynamoGraphDeploymentRequestV1beta2DefaultOptions, +) { + if input.Backends == nil { + input.Backends = []nvidiacomv1beta2.Backend{"vllm", "sglang", "trtllm"} + } + + // Hardware has no static default: it exists only when the cluster supplied some. + if input.Hardware == nil && opts.hardware != (hardwareDefaultsV1beta2{}) { + input.Hardware = &nvidiacomv1beta2.HardwareSpec{} + } + if input.Hardware != nil { + defaultHardwareSpecV1beta2(input.Hardware, opts.hardware) + } + + if input.Workload == nil { + input.Workload = &nvidiacomv1beta2.WorkloadSpec{} + } + defaultWorkloadSpecV1beta2(input.Workload) + + if input.Objective == nil { + input.Objective = &nvidiacomv1beta2.ObjectiveSpec{} + } + defaultObjectiveSpecV1beta2(input.Objective) + + // The operator image is persisted with the Request, so spec.aisimulate is created to + // carry it. + if input.AISimulate == nil && opts.aisimulateImage != "" { + input.AISimulate = &nvidiacomv1beta2.AISimulateSpec{} + } + if input.AISimulate != nil { + defaultAISimulateSpecV1beta2(input.AISimulate, opts.aisimulateImage) + } + + if input.Recommendation == nil { + input.Recommendation = &nvidiacomv1beta2.RecommendationSpec{} + } + defaultRecommendationSpecV1beta2(input.Recommendation) +} + +func defaultHardwareSpecV1beta2(hardware *nvidiacomv1beta2.HardwareSpec, defaults hardwareDefaultsV1beta2) { + if hardware.GPU == nil && defaults != (hardwareDefaultsV1beta2{}) { + hardware.GPU = &nvidiacomv1beta2.GPUHardwareSpec{} + } + if hardware.GPU != nil { + defaultGPUHardwareSpecV1beta2(hardware.GPU, defaults) + } +} + +// defaultGPUHardwareSpecV1beta2 fills each omitted value on its own, so a caller can pin the +// SKU and let admission size the budget. +func defaultGPUHardwareSpecV1beta2(gpu *nvidiacomv1beta2.GPUHardwareSpec, defaults hardwareDefaultsV1beta2) { + if gpu.SKU == nil && defaults.sku != "" { + gpu.SKU = ptr.To(defaults.sku) + } + if gpu.Budget == nil && defaults.budget > 0 { + gpu.Budget = ptr.To(defaults.budget) + } +} + +func defaultWorkloadSpecV1beta2(workload *nvidiacomv1beta2.WorkloadSpec) { + // An explicit static: {} is the opt-out, so the static defaults apply only to the object + // created here. + if workload.Static == nil { + workload.Static = &nvidiacomv1beta2.StaticWorkloadSpec{} + defaultStaticWorkloadSpecV1beta2(workload.Static) + } +} + +func defaultStaticWorkloadSpecV1beta2(static *nvidiacomv1beta2.StaticWorkloadSpec) { + static.ISL = ptr.To(defaultStaticISLV1beta2) + static.OSL = ptr.To(defaultStaticOSLV1beta2) + static.Concurrency = ptr.To(defaultStaticConcurrencyV1beta2) +} + +// defaultObjectiveSpecV1beta2 defaults the mode and gives an Optimize objective without a +// metric the default scalar target. It reads the typed objective alone: a goal.target in +// spec.aisimulate.parameters overrides the completed typed value when the input is lowered, +// so it never suppresses this default. +func defaultObjectiveSpecV1beta2(objective *nvidiacomv1beta2.ObjectiveSpec) { + if objective.Mode == "" { + objective.Mode = nvidiacomv1beta2.ObjectiveModeOptimize + } + if objective.Mode == nvidiacomv1beta2.ObjectiveModeOptimize && objective.Metric == nil { + objective.Metric = ptr.To(defaultMetricV1beta2) + } +} + +func defaultAISimulateSpecV1beta2(aisimulate *nvidiacomv1beta2.AISimulateSpec, image string) { + if aisimulate.Image == "" && image != "" { + aisimulate.Image = image + } +} + +func defaultRecommendationSpecV1beta2(recommendation *nvidiacomv1beta2.RecommendationSpec) { + if recommendation.MaxCandidates == 0 { + recommendation.MaxCandidates = defaultMaxCandidatesV1beta2 + } +} diff --git a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_helpers.go b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_helpers.go new file mode 100644 index 000000000000..ed4b3adb0848 --- /dev/null +++ b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_helpers.go @@ -0,0 +1,141 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package defaulting + +import ( + "context" + "time" + + nvidiacomv1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" + nvidiacomv1beta2 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta2" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/gpu" + admissionv1 "k8s.io/api/admission/v1" + apiequality "k8s.io/apimachinery/pkg/api/equality" + "sigs.k8s.io/controller-runtime/pkg/log" +) + +const ( + // defaultMaxHardwareBudget caps a discovered GPU budget. It matches the limit DGDR + // v1beta1 applies to auto-discovered hardware, which keeps a search from using every GPU + // in a large cluster. Set a budget explicitly to exceed it. + defaultMaxHardwareBudget int32 = 32 + + // defaultHardwareDiscoveryTimeout bounds GPU discovery inside the admission call. Keep it + // below the webhook's timeoutSeconds, or the API server gives up first and the request + // fails under the webhook's failurePolicy instead of reaching validation. + defaultHardwareDiscoveryTimeout = 5 * time.Second +) + +// HardwareDiscoverer finds the cluster's GPUs. filterSKU, when set, restricts the result to +// GPUs of that SKU. +type HardwareDiscoverer interface { + Discover(ctx context.Context, filterSKU nvidiacomv1beta1.GPUSKUType) (*gpu.GPUInfo, error) +} + +// DGDRV1beta2Options configures the defaults of v1beta2 Requests. +type DGDRV1beta2Options struct { + // AISimulateImage overrides the version-matched Dynamo planner image persisted + // into spec.aisimulate.image when the caller omits it. + AISimulateImage string + // Hardware discovers the cluster's GPUs. Nil disables hardware defaulting, and + // validation then requires the hardware to be spelled out. + Hardware HardwareDiscoverer + // HardwareDiscoveryTimeout defaults to 5 seconds. + HardwareDiscoveryTimeout time.Duration + // MaxHardwareBudget caps a discovered GPU budget; zero means 32. + MaxHardwareBudget int32 +} + +// resolveDefaults reads the environment for the values a v1beta2 spec omits. It does not +// mutate spec, and it never fails: a value the environment cannot supply stays unset and +// validating admission reports the exact field. +func (o DGDRV1beta2Options) resolveDefaults( + ctx context.Context, + operation admissionv1.Operation, + oldSpec, spec *nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec, +) dynamoGraphDeploymentRequestV1beta2DefaultOptions { + return dynamoGraphDeploymentRequestV1beta2DefaultOptions{ + hardware: o.resolveHardware(ctx, operation, oldSpec, spec), + aisimulateImage: o.AISimulateImage, + } +} + +// resolveHardware returns the SKU and budget the typed spec omits. It asks the cluster only +// when something is missing, restricted to the typed SKU when there is one. +// +// Discovery runs on CREATE and on an UPDATE that changes spec.hardware. A Request stays +// mutable and every Run snapshots it, so a caller may change or remove typed hardware before +// the next Run. An UPDATE that leaves spec.hardware unchanged, metadata-only ones included, +// never asks the cluster: a Request created while discovery had nothing to offer must not +// pick hardware up once discovery recovers, which would change its spec and generation behind +// the writer's back. oldSpec is the stored spec on UPDATE and nil otherwise. The decision reads +// the typed hardware alone, never spec.aisimulate.parameters. +func (o DGDRV1beta2Options) resolveHardware( + ctx context.Context, + operation admissionv1.Operation, + oldSpec, spec *nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec, +) hardwareDefaultsV1beta2 { + changed := operation == admissionv1.Update && oldSpec != nil && + !apiequality.Semantic.DeepEqual(oldSpec.Hardware, spec.Hardware) + if o.Hardware == nil || (operation != admissionv1.Create && !changed) { + return hardwareDefaultsV1beta2{} + } + var sku string + var hasBudget bool + if spec.Hardware != nil && spec.Hardware.GPU != nil { + if spec.Hardware.GPU.SKU != nil { + sku = *spec.Hardware.GPU.SKU + } + hasBudget = spec.Hardware.GPU.Budget != nil + } + if sku != "" && hasBudget { + return hardwareDefaultsV1beta2{} + } + + timeout := o.HardwareDiscoveryTimeout + if timeout <= 0 { + timeout = defaultHardwareDiscoveryTimeout + } + ctx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + + found, err := o.Hardware.Discover(ctx, nvidiacomv1beta1.GPUSKUType(sku)) + if err != nil { + log.FromContext(ctx).Info("hardware discovery failed; leaving hardware unresolved", "error", err.Error()) + return hardwareDefaultsV1beta2{} + } + + // The cluster facts are mapped into the v1beta2 API here: the AISimulate catalog name of + // the GPU model (empty when the catalog does not know it) and the total GPU count. + var gpus int + if found != nil { + gpus = found.GPUsPerNode * found.NodesWithGPUs + } + var defaults hardwareDefaultsV1beta2 + if sku == "" { + defaults.sku = string(gpu.KnownSKU(found)) + } + if !hasBudget && gpus > 0 { + maxBudget := o.MaxHardwareBudget + if maxBudget <= 0 { + maxBudget = defaultMaxHardwareBudget + } + defaults.budget = int32(min(gpus, int(maxBudget))) + } + return defaults +} diff --git a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_test.go b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_test.go new file mode 100644 index 000000000000..1f8c0b9aa77e --- /dev/null +++ b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_test.go @@ -0,0 +1,304 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package defaulting + +import ( + "context" + "encoding/json" + "errors" + "testing" + "time" + + nvidiacomv1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" + nvidiacomv1beta2 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta2" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/gpu" + admissionv1 "k8s.io/api/admission/v1" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" +) + +func v1beta2Ptr[T any](v T) *T { return &v } + +type fakeDiscoverer struct { + found gpu.GPUInfo + err error + calls int + filter string + timeout time.Duration +} + +func (f *fakeDiscoverer) Discover(ctx context.Context, filterSKU nvidiacomv1beta1.GPUSKUType) (*gpu.GPUInfo, error) { + f.calls++ + f.filter = string(filterSKU) + if deadline, ok := ctx.Deadline(); ok { + f.timeout = time.Until(deadline) + } + found := f.found + return &found, f.err +} + +func TestResolveHardwareV1beta2(t *testing.T) { + spec := func(sku *string, budget *int32) *nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec { + s := modelOnlySpecV1beta2() + if sku != nil || budget != nil { + s.Hardware = &nvidiacomv1beta2.HardwareSpec{GPU: &nvidiacomv1beta2.GPUHardwareSpec{SKU: sku, Budget: budget}} + } + return s + } + + t.Run("no discoverer", func(t *testing.T) { + if got := (DGDRV1beta2Options{}).resolveHardware(context.Background(), admissionv1.Create, nil, spec(nil, nil)); !(got == hardwareDefaultsV1beta2{}) { + t.Fatalf("got %+v", got) + } + }) + t.Run("complete hardware never asks the cluster", func(t *testing.T) { + d := &fakeDiscoverer{found: gpu.GPUInfo{System: "h100_sxm", GPUsPerNode: 4, NodesWithGPUs: 1}} + got := DGDRV1beta2Options{Hardware: d}.resolveHardware(context.Background(), admissionv1.Create, nil, spec(v1beta2Ptr("b200_sxm"), v1beta2Ptr(int32(2)))) + if d.calls != 0 || !(got == hardwareDefaultsV1beta2{}) { + t.Fatalf("calls=%d got=%+v", d.calls, got) + } + }) + t.Run("everything missing", func(t *testing.T) { + d := &fakeDiscoverer{found: gpu.GPUInfo{System: "h200_sxm", GPUsPerNode: 16, NodesWithGPUs: 1}} + got := DGDRV1beta2Options{Hardware: d}.resolveHardware(context.Background(), admissionv1.Create, nil, spec(nil, nil)) + if got.sku != "h200_sxm" || got.budget != 16 || d.filter != "" { + t.Fatalf("got %+v, filter %q", got, d.filter) + } + if d.timeout <= 0 || d.timeout > defaultHardwareDiscoveryTimeout { + t.Fatalf("discovery timeout = %v", d.timeout) + } + }) + t.Run("a pinned SKU filters discovery and keeps its value", func(t *testing.T) { + d := &fakeDiscoverer{found: gpu.GPUInfo{System: "b200_sxm", GPUsPerNode: 4, NodesWithGPUs: 1}} + got := DGDRV1beta2Options{Hardware: d}.resolveHardware(context.Background(), admissionv1.Create, nil, spec(v1beta2Ptr("b200_sxm"), nil)) + if d.filter != "b200_sxm" || got.sku != "" || got.budget != 4 { + t.Fatalf("got %+v, filter %q", got, d.filter) + } + }) + t.Run("a pinned budget keeps its value", func(t *testing.T) { + d := &fakeDiscoverer{found: gpu.GPUInfo{System: "h200_sxm", GPUsPerNode: 64, NodesWithGPUs: 1}} + got := DGDRV1beta2Options{Hardware: d}.resolveHardware(context.Background(), admissionv1.Create, nil, spec(nil, v1beta2Ptr(int32(3)))) + if got.sku != "h200_sxm" || got.budget != 0 { + t.Fatalf("got %+v", got) + } + }) + t.Run("the budget is capped", func(t *testing.T) { + d := &fakeDiscoverer{found: gpu.GPUInfo{System: "h200_sxm", GPUsPerNode: 1024, NodesWithGPUs: 1}} + if got := (DGDRV1beta2Options{Hardware: d}).resolveHardware(context.Background(), admissionv1.Create, nil, spec(nil, nil)); got.budget != 32 { + t.Fatalf("default cap: %+v", got) + } + if got := (DGDRV1beta2Options{Hardware: d, MaxHardwareBudget: 8}).resolveHardware(context.Background(), admissionv1.Create, nil, spec(nil, nil)); got.budget != 8 { + t.Fatalf("explicit cap: %+v", got) + } + }) + t.Run("an unknown GPU model yields no SKU", func(t *testing.T) { + d := &fakeDiscoverer{found: gpu.GPUInfo{GPUsPerNode: 8, NodesWithGPUs: 1}} + if got := (DGDRV1beta2Options{Hardware: d}).resolveHardware(context.Background(), admissionv1.Create, nil, spec(nil, nil)); got.sku != "" || got.budget != 8 { + t.Fatalf("got %+v", got) + } + }) + t.Run("a discovery failure leaves everything unset", func(t *testing.T) { + d := &fakeDiscoverer{err: errors.New("no DCGM")} + if got := (DGDRV1beta2Options{Hardware: d}).resolveHardware(context.Background(), admissionv1.Create, nil, spec(nil, nil)); !(got == hardwareDefaultsV1beta2{}) { + t.Fatalf("got %+v", got) + } + }) + t.Run("the timeout is configurable", func(t *testing.T) { + d := &fakeDiscoverer{found: gpu.GPUInfo{GPUsPerNode: 1, NodesWithGPUs: 1}} + DGDRV1beta2Options{Hardware: d, HardwareDiscoveryTimeout: time.Hour}.resolveHardware(context.Background(), admissionv1.Create, nil, spec(nil, nil)) + if d.timeout < time.Minute { + t.Fatalf("timeout = %v", d.timeout) + } + }) +} + +func TestResolveDefaultsV1beta2(t *testing.T) { + d := &fakeDiscoverer{found: gpu.GPUInfo{System: "h200_sxm", GPUsPerNode: 8, NodesWithGPUs: 1}} + got := DGDRV1beta2Options{Hardware: d, AISimulateImage: "registry.example.com/aisimulate:1.2.0"}. + resolveDefaults(context.Background(), admissionv1.Create, nil, modelOnlySpecV1beta2()) + if got.aisimulateImage != "registry.example.com/aisimulate:1.2.0" || got.hardware.sku != "h200_sxm" || got.hardware.budget != 8 { + t.Fatalf("got %+v", got) + } +} + +func v1beta2Envelope(t *testing.T, spec *nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec) *nvidiacomv1beta1.DynamoGraphDeploymentRequest { + t.Helper() + raw, err := json.Marshal(spec) + if err != nil { + t.Fatal(err) + } + dgdr := &nvidiacomv1beta1.DynamoGraphDeploymentRequest{} + dgdr.Spec.V1Beta2 = &apiextensionsv1.JSON{Raw: raw} + return dgdr +} + +func TestDefaultEnvelopeV1beta2(t *testing.T) { + options := DGDRV1beta2Options{ + AISimulateImage: "registry.example.com/aisimulate:1.2.0", + Hardware: &fakeDiscoverer{found: gpu.GPUInfo{System: "h200_sxm", GPUsPerNode: 8, NodesWithGPUs: 1}}, + } + dgdr := v1beta2Envelope(t, modelOnlySpecV1beta2()) + + if err := defaultDynamoGraphDeploymentRequestV1beta2(context.Background(), dgdr, admissionv1.Create, nil, options); err != nil { + t.Fatal(err) + } + var got nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec + if err := json.Unmarshal(dgdr.Spec.V1Beta2.Raw, &got); err != nil { + t.Fatal(err) + } + if got.Hardware == nil || got.Hardware.GPU == nil || *got.Hardware.GPU.SKU != "h200_sxm" || *got.Hardware.GPU.Budget != 8 { + t.Fatalf("hardware not defaulted: %+v", got.Hardware) + } + if got.AISimulate == nil || got.AISimulate.Image != "registry.example.com/aisimulate:1.2.0" { + t.Fatalf("image not defaulted: %+v", got.AISimulate) + } + if got.Objective == nil || got.Objective.Metric == nil { + t.Fatalf("objective metric not defaulted: %+v", got.Objective) + } + + // A second admission of the defaulted object changes nothing, byte for byte. + first := string(dgdr.Spec.V1Beta2.Raw) + if err := defaultDynamoGraphDeploymentRequestV1beta2(context.Background(), dgdr, admissionv1.Create, nil, options); err != nil { + t.Fatal(err) + } + if string(dgdr.Spec.V1Beta2.Raw) != first { + t.Fatalf("not idempotent:\n%s\n%s", first, dgdr.Spec.V1Beta2.Raw) + } +} + +func TestDefaultEnvelopeV1beta2RejectsMalformedSpec(t *testing.T) { + dgdr := &nvidiacomv1beta1.DynamoGraphDeploymentRequest{} + dgdr.Spec.V1Beta2 = &apiextensionsv1.JSON{Raw: []byte(`{"model": 3}`)} + if err := defaultDynamoGraphDeploymentRequestV1beta2(context.Background(), dgdr, admissionv1.Create, nil, DGDRV1beta2Options{}); err == nil { + t.Fatal("want a decode error") + } +} + +func hardwareSpecV1beta2(sku *string, budget *int32) *nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec { + spec := modelOnlySpecV1beta2() + if sku != nil || budget != nil { + spec.Hardware = &nvidiacomv1beta2.HardwareSpec{GPU: &nvidiacomv1beta2.GPUHardwareSpec{SKU: sku, Budget: budget}} + } + return spec +} + +func TestHardwareIsNeverDiscoveredOnOtherOperations(t *testing.T) { + for _, op := range []admissionv1.Operation{admissionv1.Delete, admissionv1.Connect, ""} { + t.Run("operation "+string(op), func(t *testing.T) { + d := &fakeDiscoverer{found: gpu.GPUInfo{System: "h200_sxm", GPUsPerNode: 8, NodesWithGPUs: 1}} + old := hardwareSpecV1beta2(v1beta2Ptr("h100_sxm"), v1beta2Ptr(int32(4))) + got := DGDRV1beta2Options{Hardware: d}.resolveHardware(context.Background(), op, old, hardwareSpecV1beta2(nil, nil)) + if got != (hardwareDefaultsV1beta2{}) || d.calls != 0 { + t.Fatalf("hardware resolved on %q: %+v, calls %d", op, got, d.calls) + } + }) + } +} + +func TestResolveHardwareByOperationV1beta2(t *testing.T) { + full := func() *nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec { + return hardwareSpecV1beta2(v1beta2Ptr("h100_sxm"), v1beta2Ptr(int32(4))) + } + tests := []struct { + name string + op admissionv1.Operation + old, spec *nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec + want hardwareDefaultsV1beta2 + wantCalls int + wantFilter string + }{ + {"create discovers everything", admissionv1.Create, nil, hardwareSpecV1beta2(nil, nil), + hardwareDefaultsV1beta2{sku: "h200_sxm", budget: 8}, 1, ""}, + {"metadata-only update without hardware does not discover", admissionv1.Update, + hardwareSpecV1beta2(nil, nil), hardwareSpecV1beta2(nil, nil), hardwareDefaultsV1beta2{}, 0, ""}, + {"update with unchanged incomplete hardware does not discover", admissionv1.Update, + hardwareSpecV1beta2(v1beta2Ptr("h100_sxm"), nil), hardwareSpecV1beta2(v1beta2Ptr("h100_sxm"), nil), + hardwareDefaultsV1beta2{}, 0, ""}, + {"update without a stored spec does not discover", admissionv1.Update, + nil, hardwareSpecV1beta2(nil, nil), hardwareDefaultsV1beta2{}, 0, ""}, + {"removing only the budget discovers only the budget", admissionv1.Update, + full(), hardwareSpecV1beta2(v1beta2Ptr("h100_sxm"), nil), + hardwareDefaultsV1beta2{budget: 8}, 1, "h100_sxm"}, + {"removing only the SKU discovers only the SKU", admissionv1.Update, + full(), hardwareSpecV1beta2(nil, v1beta2Ptr(int32(4))), + hardwareDefaultsV1beta2{sku: "h200_sxm"}, 1, ""}, + {"removing all typed hardware discovers both", admissionv1.Update, + full(), hardwareSpecV1beta2(nil, nil), + hardwareDefaultsV1beta2{sku: "h200_sxm", budget: 8}, 1, ""}, + {"changing to complete hardware does not discover", admissionv1.Update, + full(), hardwareSpecV1beta2(v1beta2Ptr("b200_sxm"), v1beta2Ptr(int32(2))), + hardwareDefaultsV1beta2{}, 0, ""}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + d := &fakeDiscoverer{found: gpu.GPUInfo{System: "h200_sxm", GPUsPerNode: 8, NodesWithGPUs: 1}} + got := DGDRV1beta2Options{Hardware: d}.resolveHardware(context.Background(), tt.op, tt.old, tt.spec) + if got != tt.want || d.calls != tt.wantCalls || (tt.wantCalls > 0 && d.filter != tt.wantFilter) { + t.Fatalf("got %+v, calls %d, filter %q; want %+v, calls %d, filter %q", + got, d.calls, d.filter, tt.want, tt.wantCalls, tt.wantFilter) + } + }) + } +} + +func TestDecodeOldSpecV1beta2(t *testing.T) { + old, err := json.Marshal(v1beta2Envelope(t, hardwareSpecV1beta2(v1beta2Ptr("h100_sxm"), nil))) + if err != nil { + t.Fatal(err) + } + got, err := decodeOldSpecV1beta2(admissionv1.Update, old) + if err != nil || got == nil || got.Hardware == nil || *got.Hardware.GPU.SKU != "h100_sxm" { + t.Fatalf("got %+v, err %v", got, err) + } + if got, err := decodeOldSpecV1beta2(admissionv1.Create, old); got != nil || err != nil { + t.Fatalf("create has no old spec: %+v, %v", got, err) + } + if got, err := decodeOldSpecV1beta2(admissionv1.Update, []byte(`{}`)); got != nil || err != nil { + t.Fatalf("an old object without the envelope has no old spec: %+v, %v", got, err) + } + if _, err := decodeOldSpecV1beta2(admissionv1.Update, []byte(`{`)); err == nil { + t.Fatal("want a decode error") + } +} + +// modelOnlySpecV1beta2 is a Request as a client sends it: only the model is named. +func modelOnlySpecV1beta2() *nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec { + spec := &nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec{} + spec.Model = &nvidiacomv1beta2.ModelSpec{Name: "meta-llama/Llama-3.1-8B-Instruct"} + return spec +} + +// An envelope that admission has nothing to add to keeps its exact bytes, even when they are +// not in canonical form. +func TestDefaultEnvelopeV1beta2LeavesUnchangedEnvelopeUntouched(t *testing.T) { + spec := modelOnlySpecV1beta2() + defaultDynamoGraphDeploymentRequestSpecV1beta2(spec, dynamoGraphDeploymentRequestV1beta2DefaultOptions{}) + raw, err := json.Marshal(spec) + if err != nil { + t.Fatal(err) + } + dgdr := v1beta2Envelope(t, spec) + dgdr.Spec.V1Beta2.Raw = append([]byte(" "), raw...) + original := string(dgdr.Spec.V1Beta2.Raw) + if err := defaultDynamoGraphDeploymentRequestV1beta2( + context.Background(), dgdr, admissionv1.Create, nil, DGDRV1beta2Options{}); err != nil { + t.Fatal(err) + } + if string(dgdr.Spec.V1Beta2.Raw) != original { + t.Fatalf("envelope was rewritten without a change: %q", dgdr.Spec.V1Beta2.Raw) + } +} diff --git a/deploy/operator/internal/webhook/request_version.go b/deploy/operator/internal/webhook/request_version.go new file mode 100644 index 000000000000..982ecc9c3558 --- /dev/null +++ b/deploy/operator/internal/webhook/request_version.go @@ -0,0 +1,35 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package webhook + +import "sigs.k8s.io/controller-runtime/pkg/webhook/admission" + +// VersionV1Beta2 is the API version whose Requests reach the webhooks as a converted v1beta1 +// object carrying spec.v1beta2. +const VersionV1Beta2 = "v1beta2" + +// RequestedVersion is the API version the client used. With matchPolicy: Equivalent the API +// server converts the object to the version the webhook registered for; RequestKind then +// carries the original kind and Kind the converted one. RequestKind is absent when no +// conversion happened. +func RequestedVersion(req admission.Request) string { + if req.RequestKind != nil { + return req.RequestKind.Version + } + return req.Kind.Version +} diff --git a/deploy/operator/internal/webhook/request_version_test.go b/deploy/operator/internal/webhook/request_version_test.go new file mode 100644 index 000000000000..837bf346d0a4 --- /dev/null +++ b/deploy/operator/internal/webhook/request_version_test.go @@ -0,0 +1,57 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package webhook + +import ( + "testing" + + admissionv1 "k8s.io/api/admission/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" +) + +// requestWithKinds returns a request whose stored kind is v1beta1, the version every webhook +// receives after conversion, and whose original request kind is requestKind (empty for none). +func requestWithKinds(requestKind string) admission.Request { + req := admission.Request{AdmissionRequest: admissionv1.AdmissionRequest{ + Kind: metav1.GroupVersionKind{Group: "nvidia.com", Version: "v1beta1", Kind: "DynamoGraphDeploymentRequest"}, + }} + if requestKind != "" { + req.RequestKind = &metav1.GroupVersionKind{Group: "nvidia.com", Version: requestKind, Kind: "DynamoGraphDeploymentRequest"} + } + return req +} + +func TestRequestedVersion(t *testing.T) { + tests := []struct { + name string + req admission.Request + want string + }{ + {"no conversion uses Kind", requestWithKinds(""), "v1beta1"}, + {"RequestKind wins after conversion", requestWithKinds("v1beta2"), "v1beta2"}, + {"RequestKind wins over a matching Kind", requestWithKinds("v1alpha1"), "v1alpha1"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := RequestedVersion(tt.req); got != tt.want { + t.Fatalf("RequestedVersion() = %q, want %q", got, tt.want) + } + }) + } +} diff --git a/deploy/operator/internal/webhook/setup/setup.go b/deploy/operator/internal/webhook/setup/setup.go index 1da053a3207a..268fe4d312c5 100644 --- a/deploy/operator/internal/webhook/setup/setup.go +++ b/deploy/operator/internal/webhook/setup/setup.go @@ -23,10 +23,11 @@ type Options struct { Config *configv1alpha1.OperatorConfiguration RuntimeConfig *commoncontroller.RuntimeConfig OperatorVersion string - // DGDRDefaultImage is the default DGDR profiler image, put into - // DGDR spec.image when unset; empty derives - // dynamo-planner:. - DGDRDefaultImage string + // DGDRV1beta1 configures the defaults of v1beta1 Requests. + DGDRV1beta1 webhookdefaulting.DGDRV1beta1Options + // DGDRV1beta2 configures the defaults of v1beta2 Requests: the AISimulate runner + // image and the cluster hardware discovery. + DGDRV1beta2 webhookdefaulting.DGDRV1beta2Options OperatorPrincipal string Gate features.Gate } @@ -104,7 +105,8 @@ func Setup(mgr ctrl.Manager, opts Options) error { return fmt.Errorf("unable to register DynamoGraphDeployment defaulting webhook: %w", err) } - dgdrDefaulter := webhookdefaulting.NewDGDRDefaulter(opts.OperatorVersion, opts.DGDRDefaultImage) + dgdrDefaulter := webhookdefaulting.NewDGDRDefaulter(opts.OperatorVersion, opts.DGDRV1beta1.DefaultImage). + WithV1beta2(opts.DGDRV1beta2) if err := dgdrDefaulter.RegisterWithManager(mgr); err != nil { return fmt.Errorf("unable to register DynamoGraphDeploymentRequest defaulting webhook: %w", err) } diff --git a/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest.go b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest.go index 96cf6c0a80d3..85e6b28a1695 100644 --- a/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest.go +++ b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest.go @@ -27,7 +27,8 @@ import ( nvidiacomv1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" "github.com/ai-dynamo/dynamo/deploy/operator/internal/dgdoverride" "github.com/ai-dynamo/dynamo/deploy/operator/internal/features" - "github.com/ai-dynamo/dynamo/deploy/operator/internal/runtimeversion" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/versionoverride" + internalwebhook "github.com/ai-dynamo/dynamo/deploy/operator/internal/webhook" apiequality "k8s.io/apimachinery/pkg/api/equality" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" k8sruntime "k8s.io/apimachinery/pkg/runtime" @@ -46,41 +47,51 @@ func NewDynamoGraphDeploymentRequestValidator() *DynamoGraphDeploymentRequestVal // dynamoGraphDeploymentRequestValidation carries DGDR-specific request state. // API values, paths, and accumulated errors remain explicit validator arguments. type dynamoGraphDeploymentRequestValidation struct { - ctx context.Context - warnings admission.Warnings + ctx context.Context + // requestVersion is the API version the client used, resolved once from the admission + // request. Every object delivered to the webhook is a converted v1beta1 object; + // spec.v1beta2 says which represented spec it carries. + requestVersion string + warnings admission.Warnings } func (v *dynamoGraphDeploymentRequestValidation) warn(message string) { v.warnings = append(v.warnings, message) } -// Validate performs stateless validation on request. ctx and request must not be nil. +// Validate performs stateless validation on request. ctx must carry the admission request and +// request must not be nil. func (v *DynamoGraphDeploymentRequestValidator) Validate( ctx context.Context, request *nvidiacomv1beta1.DynamoGraphDeploymentRequest, ) (admission.Warnings, error) { - // Legacy v1beta1 rules do not apply to a converted v1beta2 carrier. - if request.Spec.V1Beta2 != nil { - return nil, nil + req, err := admission.RequestFromContext(ctx) + if err != nil { + return nil, fmt.Errorf("get admission request: %w", err) + } + validation := &dynamoGraphDeploymentRequestValidation{ + ctx: ctx, + requestVersion: internalwebhook.RequestedVersion(req), } - - validation := &dynamoGraphDeploymentRequestValidation{ctx: ctx} allErrs := validation.validateDynamoGraphDeploymentRequest(request) return validation.warnings, invalidDynamoGraphDeploymentRequestError(request, allErrs) } -// ValidateUpdate validates newRequest against oldRequest. ctx, oldRequest, and newRequest must not be nil. +// ValidateUpdate validates newRequest against oldRequest. ctx must carry the admission request; +// oldRequest and newRequest must not be nil. func (v *DynamoGraphDeploymentRequestValidator) ValidateUpdate( ctx context.Context, oldRequest *nvidiacomv1beta1.DynamoGraphDeploymentRequest, newRequest *nvidiacomv1beta1.DynamoGraphDeploymentRequest, ) (admission.Warnings, error) { - // Legacy v1beta1 rules do not apply to a converted v1beta2 carrier. - if newRequest.Spec.V1Beta2 != nil { - return nil, nil + req, err := admission.RequestFromContext(ctx) + if err != nil { + return nil, fmt.Errorf("get admission request: %w", err) + } + validation := &dynamoGraphDeploymentRequestValidation{ + ctx: ctx, + requestVersion: internalwebhook.RequestedVersion(req), } - - validation := &dynamoGraphDeploymentRequestValidation{ctx: ctx} allErrs := validation.validateDynamoGraphDeploymentRequestUpdate(newRequest, oldRequest) validation.warnDeprecatedDGDOverrideTargets(&newRequest.Spec) validation.warnDeprecatedGPUSKU(&newRequest.Spec) @@ -99,6 +110,22 @@ func (v *dynamoGraphDeploymentRequestValidation) validateDynamoGraphDeploymentRe spec *nvidiacomv1beta1.DynamoGraphDeploymentRequestSpec, fldPath *field.Path, ) field.ErrorList { + // The requested API version decides what spec.v1beta2 means. A spec that only conversion + // may produce is rejected on CREATE. + if v.requestVersion == internalwebhook.VersionV1Beta2 && spec.V1Beta2 == nil { + // A hub/v1beta1 representation was submitted through v1beta2. + return foreignCreateErrors(fldPath, false) + } + if v.requestVersion != internalwebhook.VersionV1Beta2 && spec.V1Beta2 != nil { + // A v1beta2 carrier was submitted through v1alpha1 or v1beta1. + return foreignCreateErrors(fldPath, true) + } + // A v1beta2 Request reaches this webhook as a v1beta1 object carrying spec.v1beta2; + // the v1beta1 rules below do not apply to it. + if spec.V1Beta2 != nil { + return validateDynamoGraphDeploymentRequestEnvelopeV1beta2(spec.V1Beta2, fldPath.Child("v1beta2")) + } + allErrs := field.ErrorList{} if !features.MustGateFrom(v.ctx).Enabled(features.GPUDiscovery) && !hasManualDGDRHardware(spec.Hardware) { @@ -251,6 +278,21 @@ func (v *dynamoGraphDeploymentRequestValidation) validateDynamoGraphDeploymentRe fldPath *field.Path, oldPhase nvidiacomv1beta1.DGDRPhase, ) field.ErrorList { + // A spec viewed through the API version that does not own it is read-only: an unchanged + // spec round-trips, any other write is rejected. + if v.requestVersion == internalwebhook.VersionV1Beta2 && newSpec.V1Beta2 == nil { + return foreignUpdateVerdict(fldPath, newSpec, oldSpec) + } + if v.requestVersion != internalwebhook.VersionV1Beta2 && newSpec.V1Beta2 != nil { + return foreignUpdateVerdict(fldPath, newSpec, oldSpec) + } + // A native v1beta2 spec is unconstrained by phase. Stateless validation of the new typed + // spec runs on every CREATE and UPDATE: an unchanged spec is no reason to skip it. Ratcheting + // belongs to the generated AISimulate schema validation, which will ratchet its own result. + if newSpec.V1Beta2 != nil { + return validateDynamoGraphDeploymentRequestEnvelopeV1beta2(newSpec.V1Beta2, fldPath.Child("v1beta2")) + } + allErrs := field.ErrorList{} gpuDiscoveryEnabled := features.MustGateFrom(v.ctx).Enabled(features.GPUDiscovery) @@ -304,7 +346,7 @@ func dgdrRuntimeVersionOverrideRequired(spec *nvidiacomv1beta1.DynamoGraphDeploy if spec.Image == "" || spec.RuntimeVersionOverride != "" { return false } - _, err := runtimeversion.ParseImageVersion(spec.Image) + _, err := versionoverride.ParseImageVersion(spec.Image) return err != nil } diff --git a/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_boundary_test.go b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_boundary_test.go new file mode 100644 index 000000000000..e8fdc85c88c2 --- /dev/null +++ b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_boundary_test.go @@ -0,0 +1,39 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package validation + +import ( + "context" + "testing" + + nvidiacomv1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" +) + +// The webhook boundary protects raw conversion carriers, so a missing admission request must +// fail the call instead of counting as a native write. +func TestDGDRValidatorRequiresTheAdmissionRequest(t *testing.T) { + validator := NewDynamoGraphDeploymentRequestValidator() + request := &nvidiacomv1beta1.DynamoGraphDeploymentRequest{} + + if _, err := validator.Validate(context.Background(), request); err == nil { + t.Fatal("Validate must fail without an admission request in the context") + } + if _, err := validator.ValidateUpdate(context.Background(), request, request); err == nil { + t.Fatal("ValidateUpdate must fail without an admission request in the context") + } +} diff --git a/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_carrier_envtest_test.go b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_carrier_envtest_test.go new file mode 100644 index 000000000000..f8f8eac32574 --- /dev/null +++ b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_carrier_envtest_test.go @@ -0,0 +1,101 @@ +package validation_test + +import ( + "bytes" + "encoding/json" + "sort" + "testing" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/dynamic" + + "github.com/ai-dynamo/dynamo/deploy/operator/internal/features" +) + +// A request created through its owning API version has to stay writable through every other +// served version. A metadata-only patch through the foreign view must not be rejected, and it +// must neither rewrite the spec nor bump the generation. A default that materialises a native +// field next to a conversion carrier breaks this, so it is covered at the API-server boundary. +func TestDynamoGraphDeploymentRequestCarrierObjectsStayWritable(t *testing.T) { + t.Run("native v1beta1 written through v1beta2", func(t *testing.T) { + carrierRoundTrip(t, betaDGDRForAdmission(nil), beta2DGDRForAdmission(nil), "v1beta1") + }) + t.Run("native v1beta2 written through v1beta1", func(t *testing.T) { + carrierRoundTrip(t, beta2DGDRForAdmission(setBeta2AdmissionHardware), betaDGDRForAdmission(nil), "v1beta2") + }) +} + +func carrierRoundTrip(t *testing.T, ownerFixture, otherFixture runtime.Object, carrierKey string) { + t.Helper() + env := admissionEnv.ForTest(t) + admissionGate.set(features.Gates{GPUDiscovery: true}) + warnings := &warningRecorder{} + ownerClient := newAdmissionResourceClient(t, env, ownerFixture, "", warnings) + otherClient := newAdmissionResourceClient(t, env, otherFixture, "", warnings) + + t.Log("Create the request through the version that owns its representation") + object, _ := admissionObject(t, ownerFixture, env.Namespace(), nil) + created, err := ownerClient.Create(t.Context(), object, metav1.CreateOptions{}) + if err != nil { + t.Fatalf("create through the owning version: %v", err) + } + name := created.GetName() + + t.Log("Read it through the other version and expect the conversion carrier") + foreign, err := otherClient.Get(t.Context(), name, metav1.GetOptions{}) + if err != nil { + t.Fatalf("read through the other version: %v", err) + } + spec, _ := foreign.Object["spec"].(map[string]any) + if _, ok := spec[carrierKey]; !ok { + keys := make([]string, 0, len(spec)) + for key := range spec { + keys = append(keys, key) + } + sort.Strings(keys) + t.Fatalf("foreign view has spec keys %v, want the %s carrier", keys, carrierKey) + } + + t.Log("Patch metadata through each version") + for label, client := range map[string]dynamic.ResourceInterface{"foreign": otherClient, "owner": ownerClient} { + patch := []byte(`{"metadata":{"labels":{"carrier-probe-` + label + `":"1"}}}`) + if _, err := client.Patch(t.Context(), name, types.MergePatchType, patch, metav1.PatchOptions{}); err != nil { + t.Fatalf("metadata-only patch through the %s version: %v", label, err) + } + } + + t.Log("The spec and generation must be untouched") + after, err := ownerClient.Get(t.Context(), name, metav1.GetOptions{}) + if err != nil { + t.Fatalf("read back through the owning version: %v", err) + } + if after.GetGeneration() != created.GetGeneration() { + t.Fatalf("generation = %d, want %d: a metadata-only patch rewrote the spec", after.GetGeneration(), created.GetGeneration()) + } + + t.Log("Write the unchanged foreign view back with a full update; the spec must be byte-stable") + beforeSpec, err := json.Marshal(foreign.Object["spec"]) + if err != nil { + t.Fatalf("encode the foreign spec: %v", err) + } + current, err := otherClient.Get(t.Context(), name, metav1.GetOptions{}) + if err != nil { + t.Fatalf("read the foreign view before the update: %v", err) + } + updated, err := otherClient.Update(t.Context(), current, metav1.UpdateOptions{}) + if err != nil { + t.Fatalf("full update of the unchanged foreign view: %v", err) + } + afterSpec, err := json.Marshal(updated.Object["spec"]) + if err != nil { + t.Fatalf("encode the updated spec: %v", err) + } + if !bytes.Equal(beforeSpec, afterSpec) { + t.Fatalf("the foreign spec changed across an unchanged update:\nbefore: %s\nafter: %s", beforeSpec, afterSpec) + } + if updated.GetGeneration() != created.GetGeneration() { + t.Fatalf("generation = %d, want %d: an unchanged update rewrote the spec", updated.GetGeneration(), created.GetGeneration()) + } +} diff --git a/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_routing.go b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_routing.go new file mode 100644 index 000000000000..a0c10bc6fc31 --- /dev/null +++ b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_routing.go @@ -0,0 +1,67 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package validation + +import ( + nvidiacomv1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" + apiequality "k8s.io/apimachinery/pkg/api/equality" + "k8s.io/apimachinery/pkg/util/validation/field" +) + +// foreignCreateErrors rejects a spec that conversion alone may produce. The carrier is raw +// JSON that bypasses structural pruning; it is safe only when conversion built it from a +// native request that was validated against its own versioned schema. +func foreignCreateErrors(fldPath *field.Path, hasV1Beta2Carrier bool) field.ErrorList { + if hasV1Beta2Carrier { + return field.ErrorList{field.Forbidden( + fldPath.Child("v1beta2"), + "spec.v1beta2 is a conversion carrier that only conversion may write; create the Request through the v1beta2 API", + )} + } + return field.ErrorList{field.Forbidden( + fldPath, + "a v1beta1 representation cannot be written through the v1beta2 API; spec.v1beta1 is a conversion carrier that only conversion may write", + )} +} + +// foreignUpdateVerdict decides an UPDATE whose spec is viewed through the API version that +// does not own it: an unchanged spec round-trips, any other write is rejected. +func foreignUpdateVerdict( + fldPath *field.Path, + newSpec, oldSpec *nvidiacomv1beta1.DynamoGraphDeploymentRequestSpec, +) field.ErrorList { + if apiequality.Semantic.DeepEqual(newSpec, oldSpec) { + return nil + } + return foreignUpdateErrors(fldPath, newSpec.V1Beta2 != nil) +} + +// foreignUpdateErrors rejects a change to a spec viewed through the API version that does +// not own it. An unchanged spec never reaches this function. +func foreignUpdateErrors(fldPath *field.Path, hasV1Beta2Carrier bool) field.ErrorList { + if hasV1Beta2Carrier { + return field.ErrorList{field.Forbidden( + fldPath.Child("v1beta2"), + "spec.v1beta2 is a read-only view through this API version; update the Request through the v1beta2 API", + )} + } + return field.ErrorList{field.Forbidden( + fldPath, + "this Request is owned by v1beta1; its spec is read-only through the v1beta2 API, update it through v1beta1", + )} +} diff --git a/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_v1beta2.go b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_v1beta2.go new file mode 100644 index 000000000000..0030d86bd34c --- /dev/null +++ b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_v1beta2.go @@ -0,0 +1,78 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package validation + +import ( + "encoding/json" + + nvidiacomv1beta2 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta2" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/controller/aisimulate" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + "k8s.io/apimachinery/pkg/util/validation/field" +) + +// validateDynamoGraphDeploymentRequestEnvelopeV1beta2 validates the v1beta2 spec carried in +// spec.v1beta2 of a v1beta1 object. raw is that field and fldPath is its path; a spec that +// does not decode is reported there. Errors about the decoded spec use the v1beta2 spec's own +// field paths (spec.model, spec.aisimulate, ...), which are what the user wrote, so they are +// rooted at spec rather than at fldPath. +func validateDynamoGraphDeploymentRequestEnvelopeV1beta2( + raw *apiextensionsv1.JSON, + fldPath *field.Path, +) field.ErrorList { + var spec nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec + if err := json.Unmarshal(raw.Raw, &spec); err != nil { + return field.ErrorList{field.Invalid(fldPath, "", "must be a valid v1beta2 spec: "+err.Error())} + } + return validateDynamoGraphDeploymentRequestV1beta2(&spec) +} + +// validateDynamoGraphDeploymentRequestV1beta2 validates a defaulted v1beta2 Request spec: the +// typed fields and the image and version selection. spec.aisimulate.parameters is opaque here: the +// CRD types it as an object, and the validation of its content belongs to the schema of the +// AISimulate release the Request selects. +func validateDynamoGraphDeploymentRequestV1beta2(spec *nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec) field.ErrorList { + return validateDynamoGraphDeploymentRequestSpecV1beta2(spec, field.NewPath("spec")) +} + +// validateDynamoGraphDeploymentRequestSpecV1beta2 validates the typed rules that the source +// schema and CEL cannot express. The generated OpenAPI schema and CEL of the v1beta2 API have +// already run before conversion reaches this webhook, so they are not repeated here: native +// model presence, the model name length, backend uniqueness and the kvLoadRatio range order. +// spec and fldPath must not be nil. +func validateDynamoGraphDeploymentRequestSpecV1beta2( + spec *nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec, + fldPath *field.Path, +) field.ErrorList { + return validateAISimulateSpecV1beta2(spec.AISimulate, fldPath.Child("aisimulate")) +} + +// validateAISimulateSpecV1beta2 validates the execution artifact. Admission persists the +// operator's default image, so a Request that still has none was created while the operator +// had no default image configured. spec may be nil. +func validateAISimulateSpecV1beta2(spec *nvidiacomv1beta2.AISimulateSpec, fldPath *field.Path) field.ErrorList { + if spec == nil || spec.Image == "" { + return field.ErrorList{field.Required(fldPath.Child("image"), + "an AISimulate image is required; set it, or configure the operator's default AISimulate image")} + } + if _, err := aisimulate.ResolveVersion(spec.Image, spec.VersionOverride); err != nil { + return field.ErrorList{field.Required(fldPath.Child("versionOverride"), + "is required when spec.aisimulate.image has no parseable version tag")} + } + return nil +} diff --git a/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_v1beta2_test.go b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_v1beta2_test.go new file mode 100644 index 000000000000..385599a5dd86 --- /dev/null +++ b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_v1beta2_test.go @@ -0,0 +1,151 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package validation + +import ( + "encoding/json" + "strings" + "testing" + + nvidiacomv1beta2 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta2" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/validation/field" +) + +func v1beta2Ptr[T any](v T) *T { return &v } + +// v1beta2DefaultedSpec is a Request as it looks once the schema and admission have defaulted it. +func v1beta2DefaultedSpec() *nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec { + metric := nvidiacomv1beta2.ObjectiveMetric("ThroughputPerGPU") + return &nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec{ + Model: &nvidiacomv1beta2.ModelSpec{Name: "meta-llama/Llama-3.1-8B-Instruct"}, + Backends: []nvidiacomv1beta2.Backend{nvidiacomv1beta2.BackendVLLM, nvidiacomv1beta2.BackendSGLang, nvidiacomv1beta2.BackendTRTLLM}, + Hardware: &nvidiacomv1beta2.HardwareSpec{GPU: &nvidiacomv1beta2.GPUHardwareSpec{SKU: v1beta2Ptr("h200_sxm"), Budget: v1beta2Ptr(int32(8))}}, + Workload: &nvidiacomv1beta2.WorkloadSpec{Static: &nvidiacomv1beta2.StaticWorkloadSpec{ + ISL: v1beta2Ptr(int32(4000)), OSL: v1beta2Ptr(int32(1000)), KVLoadRatio: nvidiacomv1beta2.KVLoadRatio{0, 1}, + }}, + Objective: &nvidiacomv1beta2.ObjectiveSpec{ + Mode: nvidiacomv1beta2.ObjectiveModeOptimize, + Metric: &metric, + SLA: &nvidiacomv1beta2.ObjectiveSLASpec{TTFTMs: v1beta2Ptr(2000.0), ITLMs: v1beta2Ptr(30.0)}, + }, + AISimulate: &nvidiacomv1beta2.AISimulateSpec{Image: "registry.example.com/aisimulate:1.2.0"}, + Recommendation: &nvidiacomv1beta2.RecommendationSpec{MaxCandidates: 1}, + } +} + +func v1beta2WithParameters(spec *nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec, parameters string) *nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec { + if spec.AISimulate == nil { + spec.AISimulate = &nvidiacomv1beta2.AISimulateSpec{Image: "registry.example.com/aisimulate:1.2.0"} + } + spec.AISimulate.Parameters = &runtime.RawExtension{Raw: []byte(parameters)} + return spec +} + +func v1beta2Details(errs field.ErrorList) string { + var out []string + for _, err := range errs { + out = append(out, err.Field+": "+err.Detail) + } + return strings.Join(out, "\n") +} + +func v1beta2RequireError(t *testing.T, errs field.ErrorList, substr string) { + t.Helper() + if !strings.Contains(v1beta2Details(errs), substr) { + t.Fatalf("want an error containing %q, got:\n%s", substr, v1beta2Details(errs)) + } +} + +func v1beta2RequireNoError(t *testing.T, errs field.ErrorList) { + t.Helper() + if len(errs) > 0 { + t.Fatalf("unexpected errors:\n%s", v1beta2Details(errs)) + } +} + +func TestValidateV1beta2AcceptsADefaultedRequest(t *testing.T) { + v1beta2RequireNoError(t, validateDynamoGraphDeploymentRequestV1beta2(v1beta2DefaultedSpec())) +} + +// The webhook does not repeat what the generated schema and CEL enforce before conversion: +// native model presence, the model name length, backend uniqueness and the kvLoadRatio range +// order. The operatorenv table proves those errors end to end. +func TestValidateV1beta2LeavesSourceSchemaRulesToTheSchema(t *testing.T) { + spec := v1beta2DefaultedSpec() + spec.Model = nil + spec.Backends = []nvidiacomv1beta2.Backend{nvidiacomv1beta2.BackendVLLM, nvidiacomv1beta2.BackendVLLM} + spec.Workload.Static.KVLoadRatio = nvidiacomv1beta2.KVLoadRatio{0.9, 0.1} + v1beta2RequireNoError(t, validateDynamoGraphDeploymentRequestV1beta2(spec)) +} + +func TestValidateV1beta2ParametersAreOpaque(t *testing.T) { + for _, parameters := range []string{ + `{"workload":{"isl":0},"goal":{"target":"nope"}}`, + `[1]`, // the CRD types parameters as an object; admission does not interpret it + } { + v1beta2RequireNoError(t, validateDynamoGraphDeploymentRequestV1beta2( + v1beta2WithParameters(v1beta2DefaultedSpec(), parameters))) + } +} + +func TestValidateV1beta2AISimulateImage(t *testing.T) { + t.Run("no AISimulate section", func(t *testing.T) { + spec := v1beta2DefaultedSpec() + spec.AISimulate = nil + v1beta2RequireError(t, validateDynamoGraphDeploymentRequestV1beta2(spec), "spec.aisimulate.image: an AISimulate image is required") + }) + t.Run("parameters without an image", func(t *testing.T) { + spec := v1beta2DefaultedSpec() + spec.AISimulate = &nvidiacomv1beta2.AISimulateSpec{} + v1beta2RequireError(t, validateDynamoGraphDeploymentRequestV1beta2(spec), "spec.aisimulate.image") + }) + t.Run("custom tag needs a version", func(t *testing.T) { + spec := v1beta2DefaultedSpec() + spec.AISimulate.Image = "registry.example.com/aisimulate:custom" + v1beta2RequireError(t, validateDynamoGraphDeploymentRequestV1beta2(spec), "spec.aisimulate.versionOverride: is required when spec.aisimulate.image has no parseable version tag") + }) + t.Run("custom tag with a version", func(t *testing.T) { + spec := v1beta2DefaultedSpec() + spec.AISimulate.Image = "registry.example.com/aisimulate:custom" + spec.AISimulate.VersionOverride = "1.4.0" + v1beta2RequireNoError(t, validateDynamoGraphDeploymentRequestV1beta2(spec)) + }) +} + +func v1beta2Envelope(raw string) *apiextensionsv1.JSON { + return &apiextensionsv1.JSON{Raw: []byte(raw)} +} + +func TestValidateEnvelopeV1beta2(t *testing.T) { + path := field.NewPath("spec", "v1beta2") + + good, err := json.Marshal(v1beta2DefaultedSpec()) + if err != nil { + t.Fatal(err) + } + v1beta2RequireNoError(t, validateDynamoGraphDeploymentRequestEnvelopeV1beta2(v1beta2Envelope(string(good)), path)) + + bad := v1beta2DefaultedSpec() + bad.AISimulate = nil + raw, _ := json.Marshal(bad) + v1beta2RequireError(t, validateDynamoGraphDeploymentRequestEnvelopeV1beta2(v1beta2Envelope(string(raw)), path), "aisimulate") + + v1beta2RequireError(t, validateDynamoGraphDeploymentRequestEnvelopeV1beta2(v1beta2Envelope(`{"model": 3}`), path), "spec.v1beta2: must be a valid v1beta2 spec") +} diff --git a/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_validation_envtest_test.go b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_validation_envtest_test.go index 0aeca354f295..8a5b1c31796e 100644 --- a/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_validation_envtest_test.go +++ b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_validation_envtest_test.go @@ -18,6 +18,9 @@ package validation_test import ( + "encoding/json" + "errors" + "reflect" "strings" "testing" @@ -26,6 +29,7 @@ import ( nvidiacomv1beta2 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta2" "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" "github.com/ai-dynamo/dynamo/deploy/operator/internal/features" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/gpu" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" @@ -56,18 +60,27 @@ func TestDynamoGraphDeploymentRequestValidator_Validate(t *testing.T) { "spec":{"components":[{"name":"VllmPrefillWorker"}]} }` + discoveredH200 := gpu.GPUInfo{System: "h200_sxm", GPUsPerNode: 8, NodesWithGPUs: 1} + tests := []struct { name string request runtime.Object oldRequest runtime.Object gpuDiscovery bool seedWithoutWebhook bool - wantImage string - wantSchemaErr string - wantCELErr string - wantStatusErr string - wantWebhook []string - wantWarnings []string + // discovered and discoveryErr are what cluster GPU discovery reports while the case + // runs; beforeUpdate may change them between the seed and the update. + discovered gpu.GPUInfo + discoveryErr error + beforeUpdate func(*testing.T) + // check inspects the object the API server returned. + check func(*testing.T, *unstructured.Unstructured) + wantImage string + wantSchemaErr string + wantCELErr string + wantStatusErr string + wantWebhook []string + wantWarnings []string }{ // Source-version schema, CEL, and conversion boundaries. { @@ -104,12 +117,13 @@ func TestDynamoGraphDeploymentRequestValidator_Validate(t *testing.T) { }, { name: "v1beta2 create bypasses legacy v1beta1 validation", - request: beta2DGDRForAdmission(nil), + request: beta2DGDRForAdmission(setBeta2AdmissionHardware), }, { name: "v1beta2 update bypasses legacy v1beta1 validation", - oldRequest: beta2DGDRForAdmission(nil), + oldRequest: beta2DGDRForAdmission(setBeta2AdmissionHardware), request: beta2DGDRForAdmission(func(request *nvidiacomv1beta2.DynamoGraphDeploymentRequest) { + setBeta2AdmissionHardware(request) request.Labels = map[string]string{"updated": "true"} }), }, @@ -532,10 +546,403 @@ func TestDynamoGraphDeploymentRequestValidator_Validate(t *testing.T) { `spec.searchStrategy: Invalid value: "thorough": is incompatible with spec.backend "auto"; set spec.backend to a specific backend (sglang, trtllm, or vllm)`, }, }, + + // v1beta2: the CRD owns the unconditional defaults, admission the environment ones. + { + name: "v1beta2 model-only request gets schema and admission defaults", + request: v1beta2DGDRForAdmission(nil, nil), + gpuDiscovery: true, + discovered: discoveredH200, + check: func(t *testing.T, result *unstructured.Unstructured) { + expectV1beta2StructuralDefaults(t, result) + expectV1beta2AdmissionDefaults(t, result) + }, + }, + { + name: "v1beta2 explicit hardware is kept", + request: v1beta2DGDRForAdmission(func(spec map[string]any) { + spec["hardware"] = map[string]any{"gpu": map[string]any{"sku": "h100_sxm", "budget": int64(4)}} + }, nil), + gpuDiscovery: true, + discovered: discoveredH200, + check: func(t *testing.T, result *unstructured.Unstructured) { + if got := v1beta2String(t, result, "spec", "hardware", "gpu", "sku"); got != "h100_sxm" { + t.Fatalf("hardware.gpu.sku = %q, want h100_sxm", got) + } + if got := v1beta2Number(t, result, "spec", "hardware", "gpu", "budget"); got != 4 { + t.Fatalf("hardware.gpu.budget = %v, want 4", got) + } + }, + }, + { + name: "v1beta2 failed discovery leaves the hardware unresolved and is admitted", + request: v1beta2DGDRForAdmission(nil, nil), + gpuDiscovery: true, + discoveryErr: errors.New("no GPUs found"), + check: func(t *testing.T, result *unstructured.Unstructured) { + v1beta2Absent(t, result, "spec", "hardware") + }, + }, + { + name: "v1beta2 parameters are stored untouched and never decide hardware defaulting", + request: v1beta2DGDRForAdmission(func(spec map[string]any) { + setV1beta2SearchSpace(spec, map[string]any{"hardware_sku": "h100_sxm", "gpu_budget": int64(4)}) + }, nil), + gpuDiscovery: true, + discoveryErr: errors.New("no GPUs found"), + check: func(t *testing.T, result *unstructured.Unstructured) { + v1beta2Absent(t, result, "spec", "hardware") + if got := v1beta2String(t, result, "spec", "aisimulate", "parameters", "search_space", "hardware_sku"); got != "h100_sxm" { + t.Fatalf("parameters.search_space.hardware_sku = %q, want h100_sxm", got) + } + }, + }, + { + name: "v1beta2 parameters null does not suppress hardware discovery", + request: v1beta2DGDRForAdmission(func(spec map[string]any) { + setV1beta2SearchSpace(spec, map[string]any{"hardware_sku": nil}) + }, nil), + gpuDiscovery: true, + discovered: discoveredH200, + check: func(t *testing.T, result *unstructured.Unstructured) { + if got := v1beta2String(t, result, "spec", "hardware", "gpu", "sku"); got == "" { + t.Fatal("hardware.gpu.sku is empty: discovery must read the typed spec only") + } + }, + }, + { + name: "v1beta2 metadata-only update does not rediscover hardware after discovery recovers", + oldRequest: v1beta2DGDRForAdmission(func(spec map[string]any) { + setV1beta2SearchSpace(spec, map[string]any{"hardware_sku": "h100_sxm", "gpu_budget": int64(8)}) + }, nil), + request: v1beta2DGDRForAdmission(func(spec map[string]any) { + setV1beta2SearchSpace(spec, map[string]any{"hardware_sku": "h100_sxm", "gpu_budget": int64(8)}) + }, map[string]any{"updated": "true"}), + gpuDiscovery: true, + discoveryErr: errors.New("no GPUs found"), + beforeUpdate: func(*testing.T) { admissionHardware.set(discoveredH200, nil) }, + check: func(t *testing.T, result *unstructured.Unstructured) { + v1beta2Absent(t, result, "spec", "hardware") + if got := result.GetGeneration(); got != 1 { + t.Fatalf("generation = %d, want 1: the spec must not change", got) + } + }, + }, + { + name: "v1beta2 removing only the budget rediscovers only the budget", + oldRequest: v1beta2DGDRForAdmission(func(spec map[string]any) { + spec["hardware"] = map[string]any{"gpu": map[string]any{"sku": "h100_sxm", "budget": int64(4)}} + }, nil), + request: v1beta2DGDRForAdmission(func(spec map[string]any) { + spec["hardware"] = map[string]any{"gpu": map[string]any{"sku": "h100_sxm"}} + }, nil), + gpuDiscovery: true, + discovered: discoveredH200, + check: func(t *testing.T, result *unstructured.Unstructured) { + expectV1beta2GPU(t, result, "h100_sxm", 8) + }, + }, + { + name: "v1beta2 removing only the SKU rediscovers only the SKU", + oldRequest: v1beta2DGDRForAdmission(func(spec map[string]any) { + spec["hardware"] = map[string]any{"gpu": map[string]any{"sku": "h100_sxm", "budget": int64(4)}} + }, nil), + request: v1beta2DGDRForAdmission(func(spec map[string]any) { + spec["hardware"] = map[string]any{"gpu": map[string]any{"budget": int64(4)}} + }, nil), + gpuDiscovery: true, + discovered: discoveredH200, + check: func(t *testing.T, result *unstructured.Unstructured) { + expectV1beta2GPU(t, result, "h200_sxm", 4) + }, + }, + { + name: "v1beta2 removing all typed hardware rediscovers the SKU and the budget", + oldRequest: v1beta2DGDRForAdmission(func(spec map[string]any) { + spec["hardware"] = map[string]any{"gpu": map[string]any{"sku": "h100_sxm", "budget": int64(4)}} + }, nil), + request: v1beta2DGDRForAdmission(nil, nil), + gpuDiscovery: true, + discovered: discoveredH200, + check: func(t *testing.T, result *unstructured.Unstructured) { + expectV1beta2GPU(t, result, "h200_sxm", 8) + }, + }, + { + name: "v1beta2 empty workload receives the static model-only default", + request: v1beta2DGDRForAdmission(func(spec map[string]any) { + spec["workload"] = map[string]any{} + }, nil), + gpuDiscovery: true, + discovered: discoveredH200, + check: func(t *testing.T, result *unstructured.Unstructured) { + for key, want := range map[string]float64{"isl": 1024, "osl": 128, "concurrency": 10} { + if got := v1beta2Number(t, result, "spec", "workload", "static", key); got != want { + t.Fatalf("workload.static.%s = %v, want %v", key, got, want) + } + } + v1beta2Absent(t, result, "spec", "workload", "static", "kvLoadRatio") + }, + }, + { + name: "v1beta2 empty workload static is an opt-out and stays empty", + request: v1beta2DGDRForAdmission(func(spec map[string]any) { + spec["workload"] = map[string]any{"static": map[string]any{}} + }, nil), + gpuDiscovery: true, + discovered: discoveredH200, + check: func(t *testing.T, result *unstructured.Unstructured) { + v1beta2Empty(t, result, "spec", "workload", "static") + }, + }, + { + name: "v1beta2 empty workload static stays empty beside native parameters", + request: v1beta2DGDRForAdmission(func(spec map[string]any) { + spec["workload"] = map[string]any{"static": map[string]any{}} + spec["aisimulate"] = map[string]any{"parameters": mustJSONObject(t, `{"workload":{"isl":512,"osl":128,"concurrency":8,"request_count":100}}`)} + }, nil), + gpuDiscovery: true, + discovered: discoveredH200, + check: func(t *testing.T, result *unstructured.Unstructured) { + v1beta2Empty(t, result, "spec", "workload", "static") + v1beta2Absent(t, result, "spec", "objective", "sla") + expectV1beta2AdmissionDefaults(t, result) + }, + }, + { + name: "v1beta2 empty sla is an opt-out and is not refilled", + request: v1beta2DGDRForAdmission(func(spec map[string]any) { + spec["objective"] = map[string]any{"sla": map[string]any{}} + }, nil), + gpuDiscovery: true, + discovered: discoveredH200, + check: func(t *testing.T, result *unstructured.Unstructured) { + v1beta2Empty(t, result, "spec", "objective", "sla") + if got := v1beta2Number(t, result, "spec", "workload", "static", "isl"); got != 1024 { + t.Fatalf("workload.static.isl = %v, want the schema default 1024", got) + } + }, + }, + { + name: "v1beta2 native parameters win without rewriting the typed spec", + request: v1beta2DGDRForAdmission(func(spec map[string]any) { + spec["aisimulate"] = map[string]any{"parameters": mustJSONObject(t, `{"workload":{"isl":256}}`)} + }, nil), + gpuDiscovery: true, + discovered: discoveredH200, + check: func(t *testing.T, result *unstructured.Unstructured) { + if got := v1beta2Number(t, result, "spec", "workload", "static", "isl"); got != 1024 { + t.Fatalf("typed workload.static.isl = %v, want 1024", got) + } + if got := v1beta2Number(t, result, "spec", "aisimulate", "parameters", "workload", "isl"); got != 256 { + t.Fatalf("parameters.workload.isl = %v, want 256", got) + } + }, + }, + { + name: "v1beta2 stored invalid spec is rejected again on a metadata-only update", + oldRequest: v1beta2DGDRForAdmission(func(spec map[string]any) { + spec["aisimulate"] = map[string]any{"image": "registry.example.com/aisimulate:custom"} + }, nil), + request: v1beta2DGDRForAdmission(func(spec map[string]any) { + spec["aisimulate"] = map[string]any{"image": "registry.example.com/aisimulate:custom"} + }, map[string]any{"updated": "true"}), + seedWithoutWebhook: true, + gpuDiscovery: true, + discovered: discoveredH200, + wantWebhook: []string{ + "spec.aisimulate.versionOverride: Required value: is required when spec.aisimulate.image has no parseable version tag", + }, + }, + { + name: "v1beta2 empty objective and recommendation receive their defaults", + request: v1beta2DGDRForAdmission(func(spec map[string]any) { + spec["objective"] = map[string]any{} + spec["recommendation"] = map[string]any{} + }, nil), + gpuDiscovery: true, + discovered: discoveredH200, + check: func(t *testing.T, result *unstructured.Unstructured) { + expectV1beta2StructuralDefaults(t, result) + expectV1beta2AdmissionDefaults(t, result) + }, + }, + { + name: "v1beta2 pareto objective receives no scalar metric", + request: v1beta2DGDRForAdmission(func(spec map[string]any) { + spec["objective"] = map[string]any{"mode": "Pareto", "metrics": []any{"TTFT", "Throughput"}} + }, nil), + gpuDiscovery: true, + discovered: discoveredH200, + check: func(t *testing.T, result *unstructured.Unstructured) { + v1beta2Absent(t, result, "spec", "objective", "metric") + }, + }, + { + name: "v1beta2 explicit values are all kept", + request: v1beta2DGDRForAdmission(func(spec map[string]any) { + spec["backends"] = []any{"vllm"} + spec["hardware"] = map[string]any{"gpu": map[string]any{"sku": "h100_sxm", "budget": int64(4)}} + spec["workload"] = map[string]any{"static": map[string]any{"isl": int64(10)}} + spec["objective"] = map[string]any{"mode": "Optimize", "metric": "TTFT"} + spec["aisimulate"] = map[string]any{ + "image": "registry.example.com/aisimulate:custom", + "versionOverride": "1.4.0", + } + spec["recommendation"] = map[string]any{"maxCandidates": int64(3)} + }, nil), + gpuDiscovery: true, + discovered: discoveredH200, + check: expectV1beta2ExplicitValues, + }, + { + name: "v1beta2 duplicate backends are rejected on create", + request: v1beta2DGDRForAdmission(func(spec map[string]any) { + spec["backends"] = []any{"vllm", "vllm"} + }, nil), + gpuDiscovery: true, + discovered: discoveredH200, + wantSchemaErr: `spec.backends[1]: Duplicate value: "vllm"`, + }, + { + name: "v1beta2 missing model is rejected by the representation CEL", + request: v1beta2DGDRForAdmission(func(spec map[string]any) { delete(spec, "model") }, nil), + gpuDiscovery: true, + discovered: discoveredH200, + wantCELErr: `spec: Invalid value: exactly one of native v1beta2 fields or v1beta1 must be specified`, + }, + { + name: "v1beta2 empty model name is rejected by the schema", + request: v1beta2DGDRForAdmission(func(spec map[string]any) { + spec["model"] = map[string]any{"name": ""} + }, nil), + gpuDiscovery: true, + discovered: discoveredH200, + wantSchemaErr: `spec.model.name: Invalid value: "": spec.model.name in body should be at least 1 chars long`, + }, + { + name: "v1beta2 kvLoadRatio range needs min below max", + request: v1beta2DGDRForAdmission(func(spec map[string]any) { + spec["workload"] = map[string]any{"static": map[string]any{"kvLoadRatio": []any{0.9, 0.1}}} + }, nil), + gpuDiscovery: true, + discovered: discoveredH200, + wantCELErr: `spec.workload.static.kvLoadRatio: Invalid value: a range needs min < max`, + }, + { + name: "v1beta2 versionOverride latest is rejected by the schema", + request: v1beta2DGDRForAdmission(func(spec map[string]any) { + spec["aisimulate"] = map[string]any{"versionOverride": "latest"} + }, nil), + gpuDiscovery: true, + discovered: discoveredH200, + wantSchemaErr: `spec.aisimulate.versionOverride: Invalid value: "latest": spec.aisimulate.versionOverride in body should match '^(0|[1-9][0-9]{0,3})\.(0|[1-9][0-9]{0,3})\.(0|[1-9][0-9]{0,3})$'`, + }, + { + name: "v1beta2 versionOverride with a v prefix is rejected by the schema", + request: v1beta2DGDRForAdmission(func(spec map[string]any) { + spec["aisimulate"] = map[string]any{"versionOverride": "v0.13.0"} + }, nil), + gpuDiscovery: true, + discovered: discoveredH200, + wantSchemaErr: `spec.aisimulate.versionOverride: Invalid value: "v0.13.0": spec.aisimulate.versionOverride in body should match '^(0|[1-9][0-9]{0,3})\.(0|[1-9][0-9]{0,3})\.(0|[1-9][0-9]{0,3})$'`, + }, + { + name: "v1beta2 versionOverride 0.13.0 is accepted", + request: v1beta2DGDRForAdmission(func(spec map[string]any) { + spec["aisimulate"] = map[string]any{"versionOverride": "0.13.0"} + }, nil), + gpuDiscovery: true, + discovered: discoveredH200, + check: func(t *testing.T, result *unstructured.Unstructured) { + if got := v1beta2String(t, result, "spec", "aisimulate", "versionOverride"); got != "0.13.0" { + t.Fatalf("versionOverride = %q, want 0.13.0", got) + } + }, + }, + + // v1beta2 UPDATE: defaults stay idempotent, hardware is never rediscovered, and any + // spec change is fully validated. + { + name: "v1beta2 update without a spec change keeps the spec and generation", + oldRequest: v1beta2DGDRForAdmission(nil, nil), + request: v1beta2DGDRForAdmission(nil, map[string]any{"updated": "true"}), + gpuDiscovery: true, + discovered: discoveredH200, + check: func(t *testing.T, result *unstructured.Unstructured) { + if result.GetGeneration() != 1 { + t.Fatalf("generation = %d, want 1: defaulting must not change an unchanged spec", result.GetGeneration()) + } + expectV1beta2StructuralDefaults(t, result) + expectV1beta2AdmissionDefaults(t, result) + }, + }, + { + name: "v1beta2 metadata-only update does not pick up hardware once discovery recovers", + oldRequest: v1beta2DGDRForAdmission(nil, nil), + seedWithoutWebhook: true, // stored before hardware was validated + request: v1beta2DGDRForAdmission(nil, map[string]any{"updated": "true"}), + gpuDiscovery: true, + discoveryErr: errors.New("no GPUs found"), + beforeUpdate: func(*testing.T) { admissionHardware.set(discoveredH200, nil) }, + check: func(t *testing.T, result *unstructured.Unstructured) { + if result.GetGeneration() != 1 { + t.Fatalf("generation = %d, want 1: an update must not rewrite the spec", result.GetGeneration()) + } + v1beta2Absent(t, result, "spec", "hardware") + }, + }, + { + name: "v1beta2 native-only hardware survives a metadata-only update after discovery recovers", + oldRequest: v1beta2NativeHardwareDGDR(nil), + request: v1beta2NativeHardwareDGDR(map[string]any{"updated": "true"}), + gpuDiscovery: true, + discoveryErr: errors.New("no GPUs found"), + beforeUpdate: func(*testing.T) { admissionHardware.set(discoveredH200, nil) }, + check: func(t *testing.T, result *unstructured.Unstructured) { + if result.GetGeneration() != 1 { + t.Fatalf("generation = %d, want 1", result.GetGeneration()) + } + v1beta2Absent(t, result, "spec", "hardware") + if got := v1beta2String(t, result, "spec", "aisimulate", "parameters", "search_space", "hardware_sku"); got != "h100_sxm" { + t.Fatalf("parameters.search_space.hardware_sku = %q, want h100_sxm", got) + } + }, + }, + { + name: "v1beta2 spec change is accepted and bumps the generation", + oldRequest: v1beta2DGDRForAdmission(nil, nil), + request: v1beta2DGDRForAdmission(func(spec map[string]any) { + spec["model"] = map[string]any{"name": alternateAdmissionModel} + }, nil), + gpuDiscovery: true, + discovered: discoveredH200, + check: func(t *testing.T, result *unstructured.Unstructured) { + if result.GetGeneration() != 2 { + t.Fatalf("generation = %d, want 2", result.GetGeneration()) + } + if got := v1beta2String(t, result, "spec", "model", "name"); got != alternateAdmissionModel { + t.Fatalf("model.name = %q, want %q", got, alternateAdmissionModel) + } + }, + }, + { + name: "v1beta2 update to duplicate backends is rejected", + oldRequest: v1beta2DGDRForAdmission(nil, nil), + request: v1beta2DGDRForAdmission(func(spec map[string]any) { + spec["backends"] = []any{"sglang", "sglang"} + }, nil), + gpuDiscovery: true, + discovered: discoveredH200, + wantSchemaErr: `spec.backends[1]: Duplicate value: "sglang"`, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { + admissionHardware.set(tt.discovered, tt.discoveryErr) + t.Cleanup(func() { admissionHardware.set(gpu.GPUInfo{}, nil) }) + gates := features.Gates{GPUDiscovery: tt.gpuDiscovery} test := admissionTestCase{ object: tt.request, @@ -547,6 +954,7 @@ func TestDynamoGraphDeploymentRequestValidator_Validate(t *testing.T) { wantStatusError: tt.wantStatusErr, wantWebhookErrors: tt.wantWebhook, wantWarnings: tt.wantWarnings, + beforeUpdate: tt.beforeUpdate, } if tt.oldRequest != nil && !tt.gpuDiscovery { @@ -555,6 +963,9 @@ func TestDynamoGraphDeploymentRequestValidator_Validate(t *testing.T) { test.seedGates = &seedGates } actual := runAdmissionTest(t, test) + if tt.check != nil { + tt.check(t, actual) + } if tt.wantImage != "" { image, found, err := unstructured.NestedString(actual.Object, "spec", "image") if err != nil || !found { @@ -583,7 +994,7 @@ func TestDynamoGraphDeploymentRequestStatusRepresentationMatchesSpec(t *testing. }, { name: "v1alpha1 v1beta2 spec rejects native status", - createObject: beta2DGDRForAdmission(nil), + createObject: beta2DGDRForAdmission(setBeta2AdmissionHardware), statusObject: alphaDGDRForAdmission(nil), status: map[string]any{"state": string(nvidiacomv1alpha1.DGDRStatePending)}, }, @@ -595,13 +1006,13 @@ func TestDynamoGraphDeploymentRequestStatusRepresentationMatchesSpec(t *testing. }, { name: "v1beta1 v1beta2 spec rejects native status", - createObject: beta2DGDRForAdmission(nil), + createObject: beta2DGDRForAdmission(setBeta2AdmissionHardware), statusObject: betaDGDRForAdmission(nil), status: map[string]any{"phase": string(nvidiacomv1beta1.DGDRPhasePending)}, }, { name: "v1beta2 native spec rejects v1beta1 status", - createObject: beta2DGDRForAdmission(nil), + createObject: beta2DGDRForAdmission(setBeta2AdmissionHardware), statusObject: beta2DGDRForAdmission(nil), status: map[string]any{"v1beta1": map[string]any{"phase": string(nvidiacomv1beta1.DGDRPhasePending)}}, }, @@ -652,6 +1063,7 @@ func TestDynamoGraphDeploymentRequestV1Beta2EmptyStatusSurvivesV1Alpha1(t *testi warnings := &warningRecorder{} beta2Fixture := beta2DGDRForAdmission(func(request *nvidiacomv1beta2.DynamoGraphDeploymentRequest) { request.Name = "empty-status-alpha-bridge" + setBeta2AdmissionHardware(request) }) beta2Client := newAdmissionResourceClient(t, env, beta2Fixture, "", warnings) object, _ := admissionObject(t, beta2Fixture, env.Namespace(), nil) @@ -764,3 +1176,207 @@ func beta2DGDRForAdmission( } return request } + +const v1beta2AdmissionModel = "Qwen/Qwen3-0.6B" + +// v1beta2NativeHardwareDGDR supplies hardware only through native AISimulate parameters. +func v1beta2NativeHardwareDGDR(labels map[string]any) *unstructured.Unstructured { + return v1beta2DGDRForAdmission(func(spec map[string]any) { + spec["aisimulate"] = map[string]any{ + "image": admissionAISimulateImage, + "parameters": map[string]any{ + "search_space": map[string]any{"hardware_sku": "h100_sxm", "gpu_budget": int64(8)}, + }, + } + }, labels) +} + +// v1beta2DGDRForAdmission builds a v1beta2 Request as a client sends it, before the CRD +// defaults. It is unstructured on purpose: an omitted field, +// an explicit empty object and a null are different requests, and typed Go cannot tell them +// apart. +func v1beta2DGDRForAdmission(mutate func(spec map[string]any), labels map[string]any) *unstructured.Unstructured { + spec := map[string]any{"model": map[string]any{"name": v1beta2AdmissionModel}} + if mutate != nil { + mutate(spec) + } + metadata := map[string]any{"name": "test-v1beta2-dgdr", "namespace": "default"} + if labels != nil { + metadata["labels"] = labels + } + return &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": "nvidia.com/v1beta2", + "kind": "DynamoGraphDeploymentRequest", + "metadata": metadata, + "spec": spec, + }} +} + +// mustJSONObject decodes a native AISimulate parameters document. +func mustJSONObject(t *testing.T, document string) map[string]any { + t.Helper() + var value map[string]any + if err := json.Unmarshal([]byte(document), &value); err != nil { + t.Fatalf("decode %q: %v", document, err) + } + return value +} + +func v1beta2Field(t *testing.T, obj *unstructured.Unstructured, path ...string) (any, bool) { + t.Helper() + value, found, err := unstructured.NestedFieldNoCopy(obj.Object, path...) + if err != nil { + t.Fatalf("read %v: %v", path, err) + } + return value, found +} + +func v1beta2Number(t *testing.T, obj *unstructured.Unstructured, path ...string) float64 { + t.Helper() + value, found := v1beta2Field(t, obj, path...) + if !found { + t.Fatalf("%v is not set", path) + } + switch n := value.(type) { + case int64: + return float64(n) + case float64: + return n + default: + t.Fatalf("%v = %#v, want a number", path, value) + return 0 + } +} + +func v1beta2String(t *testing.T, obj *unstructured.Unstructured, path ...string) string { + t.Helper() + value, found := v1beta2Field(t, obj, path...) + if !found { + t.Fatalf("%v is not set", path) + } + text, ok := value.(string) + if !ok { + t.Fatalf("%v = %#v, want a string", path, value) + } + return text +} + +func v1beta2Absent(t *testing.T, obj *unstructured.Unstructured, path ...string) { + t.Helper() + if value, found := v1beta2Field(t, obj, path...); found { + t.Fatalf("%v = %#v, want it absent", path, value) + } +} + +func v1beta2Empty(t *testing.T, obj *unstructured.Unstructured, path ...string) { + t.Helper() + value, found := v1beta2Field(t, obj, path...) + if !found || !reflect.DeepEqual(value, map[string]any{}) { + t.Fatalf("%v = %#v (found=%v), want an empty object", path, value, found) + } +} + +// expectV1beta2StructuralDefaults checks the defaults admission supplies to a model-only Request. +func expectV1beta2StructuralDefaults(t *testing.T, obj *unstructured.Unstructured) { + t.Helper() + for key, want := range map[string]float64{"isl": 1024, "osl": 128, "concurrency": 10} { + if got := v1beta2Number(t, obj, "spec", "workload", "static", key); got != want { + t.Fatalf("workload.static.%s = %v, want %v", key, got, want) + } + } + v1beta2Absent(t, obj, "spec", "workload", "static", "kvLoadRatio") + v1beta2Absent(t, obj, "spec", "objective", "sla") + if got := v1beta2String(t, obj, "spec", "objective", "mode"); got != "Optimize" { + t.Fatalf("objective.mode = %q, want Optimize", got) + } + if got, _ := v1beta2Field(t, obj, "spec", "backends"); !reflect.DeepEqual(got, []any{"vllm", "sglang", "trtllm"}) { + t.Fatalf("backends = %#v, want vllm, sglang and trtllm", got) + } + if got := v1beta2Number(t, obj, "spec", "recommendation", "maxCandidates"); got != 1 { + t.Fatalf("recommendation.maxCandidates = %v, want 1", got) + } +} + +// expectV1beta2ExplicitValues checks that every value the explicit-values row sent survived. +func expectV1beta2ExplicitValues(t *testing.T, obj *unstructured.Unstructured) { + t.Helper() + if got, _ := v1beta2Field(t, obj, "spec", "backends"); !reflect.DeepEqual(got, []any{"vllm"}) { + t.Fatalf("backends = %#v, want only vllm", got) + } + for _, want := range []struct { + path []string + value float64 + }{ + {[]string{"spec", "hardware", "gpu", "budget"}, 4}, + {[]string{"spec", "workload", "static", "isl"}, 10}, + {[]string{"spec", "recommendation", "maxCandidates"}, 3}, + } { + if got := v1beta2Number(t, obj, want.path...); got != want.value { + t.Fatalf("%v = %v, want %v", want.path, got, want.value) + } + } + for _, want := range []struct { + path []string + value string + }{ + {[]string{"spec", "hardware", "gpu", "sku"}, "h100_sxm"}, + {[]string{"spec", "objective", "mode"}, "Optimize"}, + {[]string{"spec", "objective", "metric"}, "TTFT"}, + {[]string{"spec", "aisimulate", "image"}, "registry.example.com/aisimulate:custom"}, + {[]string{"spec", "aisimulate", "versionOverride"}, "1.4.0"}, + } { + if got := v1beta2String(t, obj, want.path...); got != want.value { + t.Fatalf("%v = %q, want %q", want.path, got, want.value) + } + } + // A static workload that names only isl is not filled in. + v1beta2Absent(t, obj, "spec", "workload", "static", "osl") + v1beta2Absent(t, obj, "spec", "workload", "static", "concurrency") +} + +// expectV1beta2AdmissionDefaults checks what admission supplies from the environment. +func expectV1beta2AdmissionDefaults(t *testing.T, obj *unstructured.Unstructured) { + t.Helper() + if got := v1beta2String(t, obj, "spec", "objective", "metric"); got != "Throughput" { + t.Fatalf("objective.metric = %q, want Throughput", got) + } + if got := v1beta2String(t, obj, "spec", "aisimulate", "image"); got != admissionAISimulateImage { + t.Fatalf("aisimulate.image = %q, want %q", got, admissionAISimulateImage) + } + if got := v1beta2String(t, obj, "spec", "hardware", "gpu", "sku"); got != "h200_sxm" { + t.Fatalf("hardware.gpu.sku = %q, want h200_sxm", got) + } + if got := v1beta2Number(t, obj, "spec", "hardware", "gpu", "budget"); got != 8 { + t.Fatalf("hardware.gpu.budget = %v, want 8", got) + } +} + +// setV1beta2SearchSpace sets spec.aisimulate.parameters.search_space to searchSpace, replacing +// any parameters the fixture carried. +func setV1beta2SearchSpace(spec map[string]any, searchSpace map[string]any) { + aisimulate, _ := spec["aisimulate"].(map[string]any) + if aisimulate == nil { + aisimulate = map[string]any{} + spec["aisimulate"] = aisimulate + } + aisimulate["parameters"] = map[string]any{"search_space": searchSpace} +} + +// setBeta2AdmissionHardware names the hardware, so fixtures do not depend on GPU discovery. +func setBeta2AdmissionHardware(request *nvidiacomv1beta2.DynamoGraphDeploymentRequest) { + sku, budget := "h100_sxm", int32(4) + request.Spec.Hardware = &nvidiacomv1beta2.HardwareSpec{ + GPU: &nvidiacomv1beta2.GPUHardwareSpec{SKU: &sku, Budget: &budget}, + } +} + +// expectV1beta2GPU asserts the typed GPU hardware the API server stored. +func expectV1beta2GPU(t *testing.T, result *unstructured.Unstructured, sku string, budget float64) { + t.Helper() + if got := v1beta2String(t, result, "spec", "hardware", "gpu", "sku"); got != sku { + t.Fatalf("hardware.gpu.sku = %q, want %q", got, sku) + } + if got := v1beta2Number(t, result, "spec", "hardware", "gpu", "budget"); got != budget { + t.Fatalf("hardware.gpu.budget = %v, want %v", got, budget) + } +} diff --git a/deploy/operator/internal/webhook/validation/planner_runtime_features.go b/deploy/operator/internal/webhook/validation/planner_runtime_features.go index 1e772c501f72..c61ff68ec318 100644 --- a/deploy/operator/internal/webhook/validation/planner_runtime_features.go +++ b/deploy/operator/internal/webhook/validation/planner_runtime_features.go @@ -13,7 +13,7 @@ import ( "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo" runtimefeatures "github.com/ai-dynamo/dynamo/deploy/operator/internal/features/runtime" - "github.com/ai-dynamo/dynamo/deploy/operator/internal/runtimeversion" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/versionoverride" "k8s.io/apimachinery/pkg/util/validation/field" ) @@ -117,7 +117,7 @@ func rolePodTemplatesPlannerRuntimeErrorDetail(planners []plannerRuntimeContract minimum := runtimefeatures.PlannerDGDComponentStatus.MinRuntimeVersion.String() for _, planner := range planners { - version, err := runtimeversion.Resolve(planner.image, planner.runtimeVersionOverride) + version, err := versionoverride.Resolve(planner.image, planner.runtimeVersionOverride) if err != nil { return fmt.Sprintf( "role-specific PodTemplates require every planner component to use Dynamo runtime %s or later; planner component %q has no resolvable runtime version: %v", diff --git a/deploy/operator/internal/webhook/validation/shared_helpers.go b/deploy/operator/internal/webhook/validation/shared_helpers.go index 053c8e1f0041..19e302c7bfac 100644 --- a/deploy/operator/internal/webhook/validation/shared_helpers.go +++ b/deploy/operator/internal/webhook/validation/shared_helpers.go @@ -30,7 +30,7 @@ import ( "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/epp" "github.com/ai-dynamo/dynamo/deploy/operator/internal/features" runtimefeatures "github.com/ai-dynamo/dynamo/deploy/operator/internal/features/runtime" - "github.com/ai-dynamo/dynamo/deploy/operator/internal/runtimeversion" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/versionoverride" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/validation/field" @@ -153,7 +153,7 @@ func runtimeVersionOverrideRequired(image, override string) bool { if override != "" { return false } - _, err := runtimeversion.ParseImageVersion(image) + _, err := versionoverride.ParseImageVersion(image) return err != nil } @@ -196,7 +196,7 @@ func eppRuntimeCompatibilityError(contract eppRuntimeContract, eppConfigPath *fi return nil } - version, err := runtimeversion.Resolve(contract.image, contract.runtimeVersionOverride) + version, err := versionoverride.Resolve(contract.image, contract.runtimeVersionOverride) if err != nil { return nil } diff --git a/deploy/operator/internal/webhook/validation/suite_envtest_test.go b/deploy/operator/internal/webhook/validation/suite_envtest_test.go index 12e4485f2f6b..f39b4e38bc02 100644 --- a/deploy/operator/internal/webhook/validation/suite_envtest_test.go +++ b/deploy/operator/internal/webhook/validation/suite_envtest_test.go @@ -20,8 +20,10 @@ import ( nvidiacomv1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" nvidiacomv1beta2 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta2" "github.com/ai-dynamo/dynamo/deploy/operator/internal/features" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/gpu" "github.com/ai-dynamo/dynamo/deploy/operator/internal/provideroverride" "github.com/ai-dynamo/dynamo/deploy/operator/internal/testing/operatorenv" + webhookdefaulting "github.com/ai-dynamo/dynamo/deploy/operator/internal/webhook/defaulting" webhooksetup "github.com/ai-dynamo/dynamo/deploy/operator/internal/webhook/setup" grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" @@ -43,6 +45,7 @@ const ( frontendImage150 = "registry.example/dynamo-frontend:1.5.0" legacyEPPImage140 = "registry.example/epp-image:1.4.0" legacySeedUsername = "operatorenv-legacy-seeder" + admissionAISimulateImage = "nvcr.io/nvidia/ai-dynamo/dynamo-planner:1.6.0" ) func groveProviderOverride(target, value string) *nvidiacomv1beta1.ProviderOverride { @@ -65,6 +68,9 @@ var ( // Fixture barriers observe the same cache used by webhook validation. admissionReader client.Reader + // Hardware that the v1beta2 defaulter discovers; tests set it before each case. + admissionHardware = &fakeHardwareDiscoverer{} + // Admission cases must remain sequential because they share this gate and a cluster-scoped topology fixture. admissionGate = &mutableFeatureGate{} admissionEnv = operatorenv.New(operatorenv.Options{ @@ -91,9 +97,33 @@ func setupAdmissionWebhooks(mgr ctrl.Manager, opts operatorenv.WebhookSetupOptio OperatorVersion: opts.OperatorVersion, OperatorPrincipal: opts.OperatorPrincipal, Gate: admissionGate, + DGDRV1beta2: webhookdefaulting.DGDRV1beta2Options{ + Hardware: admissionHardware, + }, }) } +// fakeHardwareDiscoverer stands in for the cluster's GPU discovery. Cases run sequentially, +// so a plain mutex-guarded value is enough. +type fakeHardwareDiscoverer struct { + mu sync.Mutex + found gpu.GPUInfo + err error +} + +func (f *fakeHardwareDiscoverer) set(found gpu.GPUInfo, err error) { + f.mu.Lock() + defer f.mu.Unlock() + f.found, f.err = found, err +} + +func (f *fakeHardwareDiscoverer) Discover(context.Context, nvidiacomv1beta1.GPUSKUType) (*gpu.GPUInfo, error) { + f.mu.Lock() + defer f.mu.Unlock() + found := f.found + return &found, f.err +} + type mutableFeatureGate struct { mu sync.RWMutex gates features.Gate @@ -122,6 +152,10 @@ type admissionTestCase struct { terminating bool username string + // beforeUpdate runs after the old state is seeded and before the update is submitted. Use + // it to change what the webhooks see between the two writes, such as hardware discovery. + beforeUpdate func(*testing.T) + wantSchemaError string wantCELError string wantStatusError string @@ -166,6 +200,10 @@ func runAdmissionTest(t *testing.T, test admissionTestCase) *unstructured.Unstru old = beginTermination(t, seedClient, old) } + if test.beforeUpdate != nil { + test.beforeUpdate(t) + } + t.Log("Submit the update request through the Kubernetes API server") warnings.clear() oldFixture, _ := admissionObject(t, test.oldObject, env.Namespace(), nil) diff --git a/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md b/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md index 6331c345674b..899252723888 100644 --- a/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md +++ b/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md @@ -3242,8 +3242,8 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | -| `image` _string_ | Image selects a Dynamo-compatible AISimulate runner. When omitted, admission
and execution defaults are resolved outside this API. | | MinLength: 1
Optional: \{\}
| -| `versionOverride` _string_ | VersionOverride declares the AISimulate compatibility version when it
cannot be inferred from the effective image. | | MinLength: 1
Optional: \{\}
| +| `image` _string_ | Image selects a Dynamo-compatible AISimulate runner. When omitted, admission
persists the configured override or a version-matched Dynamo planner image. | | MinLength: 1
Optional: \{\}
| +| `versionOverride` _string_ | VersionOverride declares the AISimulate compatibility version as three numeric parts
(for example 0.13.0), not a semver version: no v prefix, pre-release or build suffix. It is required
when the image tag is not a version and wins over a version in the tag. | | MinLength: 1
Pattern: `^(0\|[1-9][0-9]\{0,3\})\.(0\|[1-9][0-9]\{0,3\})\.(0\|[1-9][0-9]\{0,3\})$`
Optional: \{\}
| | `parameters` _[RawExtension](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#rawextension-runtime-pkg)_ | Parameters contains native AISimulate input. Field names and nesting use
the upstream AISimulate JSON contract, including snake_case names. | | Type: object
Optional: \{\}
| @@ -3438,8 +3438,8 @@ _Appears in:_ | `model` _[ModelSpec](#modelspec)_ | Model identifies the model whose deployment configurations are evaluated. | | Optional: \{\}
| | `backends` _[Backend](#backend) array_ | Backends lists one or more inference backends searched by AISimulate and used by generated candidates. | | Enum: [vllm sglang trtllm]
MinItems: 1
Optional: \{\}
| | `hardware` _[HardwareSpec](#v1beta2-hardwarespec)_ | Hardware bounds the accelerator configurations evaluated by the search.
Admission discovers any omitted hardware values from the cluster. | | Optional: \{\}
| -| `workload` _[WorkloadSpec](#v1beta2-workloadspec)_ | Workload defines the synthetic traffic model used for every candidate evaluation. | | Optional: \{\}
| -| `objective` _[ObjectiveSpec](#objectivespec)_ | Objective defines scalar optimization or a multi-objective Pareto search.
When omitted, the schema supplies the default Optimize mode and SLA;
admission supplies the conditional scalar metric. | | Optional: \{\}
| +| `workload` _[WorkloadSpec](#v1beta2-workloadspec)_ | Workload defines the synthetic traffic model used for every candidate evaluation.
When omitted, admission supplies a static shape of ISL 1024, OSL 128 and concurrency 10;
an explicit empty workload receives the same shape from the schema. An explicit
workload.static: \{\} opts out of that shape, leaving the workload to
spec.aisimulate.parameters.workload. spec.aisimulate.parameters.workload overrides the
fields it names.
The default evaluates a fixed concurrency, so it creates neither sweep.max_trials nor a
KV-load ratio search. | | Optional: \{\}
| +| `objective` _[ObjectiveSpec](#objectivespec)_ | Objective defines scalar optimization or a multi-objective Pareto search.
When omitted, admission supplies the object. Its mode defaults to Optimize, and admission
supplies the conditional scalar metric. | | Optional: \{\}
| | `aisimulate` _[AISimulateSpec](#aisimulatespec)_ | AISimulate selects the execution artifact and supplies native AISimulate input. | | Optional: \{\}
| | `recommendation` _[RecommendationSpec](#recommendationspec)_ | Recommendation controls bounded projection into DGDC resources. | | Optional: \{\}
| @@ -3480,8 +3480,8 @@ _Appears in:_ | `model` _[ModelSpec](#modelspec)_ | Model identifies the model whose deployment configurations are evaluated. | | Optional: \{\}
| | `backends` _[Backend](#backend) array_ | Backends lists one or more inference backends searched by AISimulate and used by generated candidates. | | Enum: [vllm sglang trtllm]
MinItems: 1
Optional: \{\}
| | `hardware` _[HardwareSpec](#v1beta2-hardwarespec)_ | Hardware bounds the accelerator configurations evaluated by the search.
Admission discovers any omitted hardware values from the cluster. | | Optional: \{\}
| -| `workload` _[WorkloadSpec](#v1beta2-workloadspec)_ | Workload defines the synthetic traffic model used for every candidate evaluation. | | Optional: \{\}
| -| `objective` _[ObjectiveSpec](#objectivespec)_ | Objective defines scalar optimization or a multi-objective Pareto search.
When omitted, the schema supplies the default Optimize mode and SLA;
admission supplies the conditional scalar metric. | | Optional: \{\}
| +| `workload` _[WorkloadSpec](#v1beta2-workloadspec)_ | Workload defines the synthetic traffic model used for every candidate evaluation.
When omitted, admission supplies a static shape of ISL 1024, OSL 128 and concurrency 10;
an explicit empty workload receives the same shape from the schema. An explicit
workload.static: \{\} opts out of that shape, leaving the workload to
spec.aisimulate.parameters.workload. spec.aisimulate.parameters.workload overrides the
fields it names.
The default evaluates a fixed concurrency, so it creates neither sweep.max_trials nor a
KV-load ratio search. | | Optional: \{\}
| +| `objective` _[ObjectiveSpec](#objectivespec)_ | Objective defines scalar optimization or a multi-objective Pareto search.
When omitted, admission supplies the object. Its mode defaults to Optimize, and admission
supplies the conditional scalar metric. | | Optional: \{\}
| | `aisimulate` _[AISimulateSpec](#aisimulatespec)_ | AISimulate selects the execution artifact and supplies native AISimulate input. | | Optional: \{\}
| | `recommendation` _[RecommendationSpec](#recommendationspec)_ | Recommendation controls bounded projection into DGDC resources. | | Optional: \{\}
| | `v1beta1` _[JSON](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#json-v1-apiextensions-k8s-io)_ | V1Beta1 losslessly preserves a native v1beta1 spec while this object is
represented through v1beta2. It is mutually exclusive with native fields.
This field is a read-only conversion view; modify the request through the
native v1beta1 API. | | Type: object
Optional: \{\}
| @@ -3543,8 +3543,8 @@ _Appears in:_ | `model` _[ModelSpec](#modelspec)_ | Model identifies the model whose deployment configurations are evaluated. | | Optional: \{\}
| | `backends` _[Backend](#backend) array_ | Backends lists one or more inference backends searched by AISimulate and used by generated candidates. | | Enum: [vllm sglang trtllm]
MinItems: 1
Optional: \{\}
| | `hardware` _[HardwareSpec](#v1beta2-hardwarespec)_ | Hardware bounds the accelerator configurations evaluated by the search.
Admission discovers any omitted hardware values from the cluster. | | Optional: \{\}
| -| `workload` _[WorkloadSpec](#v1beta2-workloadspec)_ | Workload defines the synthetic traffic model used for every candidate evaluation. | | Optional: \{\}
| -| `objective` _[ObjectiveSpec](#objectivespec)_ | Objective defines scalar optimization or a multi-objective Pareto search.
When omitted, the schema supplies the default Optimize mode and SLA;
admission supplies the conditional scalar metric. | | Optional: \{\}
| +| `workload` _[WorkloadSpec](#v1beta2-workloadspec)_ | Workload defines the synthetic traffic model used for every candidate evaluation.
When omitted, admission supplies a static shape of ISL 1024, OSL 128 and concurrency 10;
an explicit empty workload receives the same shape from the schema. An explicit
workload.static: \{\} opts out of that shape, leaving the workload to
spec.aisimulate.parameters.workload. spec.aisimulate.parameters.workload overrides the
fields it names.
The default evaluates a fixed concurrency, so it creates neither sweep.max_trials nor a
KV-load ratio search. | | Optional: \{\}
| +| `objective` _[ObjectiveSpec](#objectivespec)_ | Objective defines scalar optimization or a multi-objective Pareto search.
When omitted, admission supplies the object. Its mode defaults to Optimize, and admission
supplies the conditional scalar metric. | | Optional: \{\}
| | `aisimulate` _[AISimulateSpec](#aisimulatespec)_ | AISimulate selects the execution artifact and supplies native AISimulate input. | | Optional: \{\}
| | `recommendation` _[RecommendationSpec](#recommendationspec)_ | Recommendation controls bounded projection into DGDC resources. | | Optional: \{\}
| @@ -3686,8 +3686,8 @@ _Appears in:_ | `model` _[ModelSpec](#modelspec)_ | Model identifies the model whose deployment configurations are evaluated. | | Optional: \{\}
| | `backends` _[Backend](#backend) array_ | Backends lists one or more inference backends searched by AISimulate and used by generated candidates. | | Enum: [vllm sglang trtllm]
MinItems: 1
Optional: \{\}
| | `hardware` _[HardwareSpec](#v1beta2-hardwarespec)_ | Hardware bounds the accelerator configurations evaluated by the search.
Admission discovers any omitted hardware values from the cluster. | | Optional: \{\}
| -| `workload` _[WorkloadSpec](#v1beta2-workloadspec)_ | Workload defines the synthetic traffic model used for every candidate evaluation. | | Optional: \{\}
| -| `objective` _[ObjectiveSpec](#objectivespec)_ | Objective defines scalar optimization or a multi-objective Pareto search.
When omitted, the schema supplies the default Optimize mode and SLA;
admission supplies the conditional scalar metric. | | Optional: \{\}
| +| `workload` _[WorkloadSpec](#v1beta2-workloadspec)_ | Workload defines the synthetic traffic model used for every candidate evaluation.
When omitted, admission supplies a static shape of ISL 1024, OSL 128 and concurrency 10;
an explicit empty workload receives the same shape from the schema. An explicit
workload.static: \{\} opts out of that shape, leaving the workload to
spec.aisimulate.parameters.workload. spec.aisimulate.parameters.workload overrides the
fields it names.
The default evaluates a fixed concurrency, so it creates neither sweep.max_trials nor a
KV-load ratio search. | | Optional: \{\}
| +| `objective` _[ObjectiveSpec](#objectivespec)_ | Objective defines scalar optimization or a multi-objective Pareto search.
When omitted, admission supplies the object. Its mode defaults to Optimize, and admission
supplies the conditional scalar metric. | | Optional: \{\}
| | `aisimulate` _[AISimulateSpec](#aisimulatespec)_ | AISimulate selects the execution artifact and supplies native AISimulate input. | | Optional: \{\}
| | `recommendation` _[RecommendationSpec](#recommendationspec)_ | Recommendation controls bounded projection into DGDC resources. | | Optional: \{\}
| @@ -3760,9 +3760,9 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | | `mode` _[ObjectiveMode](#objectivemode)_ | Mode selects scalar optimization or a Pareto search. | Optimize | Enum: [Optimize Pareto]
Optional: \{\}
| -| `metric` _[ObjectiveMetric](#objectivemetric)_ | Metric is the scalar target when mode is Optimize.
When omitted in Optimize mode, admission defaults it to ThroughputPerGPU. | | Enum: [Throughput ThroughputPerGPU ThroughputPerUser TTFT E2ELatency Goodput GoodputPerGPU]
Optional: \{\}
| +| `metric` _[ObjectiveMetric](#objectivemetric)_ | Metric is the scalar target when mode is Optimize.
When omitted in Optimize mode, admission defaults it to Throughput. | | Enum: [Throughput ThroughputPerGPU ThroughputPerUser TTFT E2ELatency Goodput GoodputPerGPU]
Optional: \{\}
| | `metrics` _[ObjectiveMetric](#objectivemetric) array_ | Metrics lists two or more distinct scalar targets when mode is Pareto. | | Enum: [Throughput ThroughputPerGPU ThroughputPerUser TTFT E2ELatency Goodput GoodputPerGPU]
MaxItems: 7
MinItems: 2
Optional: \{\}
| -| `sla` _[ObjectiveSLASpec](#objectiveslaspec)_ | SLA defines per-request latency bounds used to calculate goodput. | \{ itlMs:30 ttftMs:2000 \} | Optional: \{\}
| +| `sla` _[ObjectiveSLASpec](#objectiveslaspec)_ | SLA defines per-request latency bounds used to calculate goodput. When omitted, no
bounds apply. spec.aisimulate.parameters.goal.sla overrides the bounds it names. | | Optional: \{\}
| #### RecommendationSpec @@ -3855,7 +3855,7 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | -| `static` _[StaticWorkloadSpec](#staticworkloadspec)_ | Static defines one of AISimulate's synthetic workload shapes. When omitted,
it searches the full KV-load ratio range at ISL 4000 and OSL 1000.
An explicit empty object opts out of this underlay. | \{ isl:4000 kvLoadRatio:[0 1] osl:1000 \} | Optional: \{\}
| +| `static` _[StaticWorkloadSpec](#staticworkloadspec)_ | Static defines one of AISimulate's synthetic workload shapes. Its fields are not
defaulted individually: the default belongs to spec.workload as a whole, so an
explicit workload object keeps exactly the fields it names. | \{ concurrency:10 isl:1024 osl:128 \} | Optional: \{\}
| diff --git a/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx b/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx index 571fd8ba74d1..019faa3c58ee 100644 --- a/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx +++ b/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx @@ -4531,13 +4531,13 @@ AISimulateSpec selects the AISimulate execution artifact and supplies native inp **Appears in:** [DynamoGraphDeploymentRequestNativeSpec](#v1beta2-dynamographdeploymentrequestnativespec), [DynamoGraphDeploymentRequestSpec](#v1beta2-dynamographdeploymentrequestspec), [DynamoGraphDeploymentRunSpec](#dynamographdeploymentrunspec), [NativeSearchInput](#nativesearchinput) -Image selects a Dynamo-compatible AISimulate runner. When omitted, admission and execution defaults are resolved outside this API. +Image selects a Dynamo-compatible AISimulate runner. When omitted, admission persists the configured override or a version-matched Dynamo planner image. **Validation:** MinLength: 1 Optional: \{\} -VersionOverride declares the AISimulate compatibility version when it cannot be inferred from the effective image. -**Validation:** MinLength: 1 Optional: \{\} +VersionOverride declares the AISimulate compatibility version as three numeric parts (for example 0.13.0), not a semver version: no v prefix, pre-release or build suffix. It is required when the image tag is not a version and wins over a version in the tag. +**Validation:** MinLength: 1 Pattern: `^(0|[1-9][0-9]\{0,3\})\.(0|[1-9][0-9]\{0,3\})\.(0|[1-9][0-9]\{0,3\})$` Optional: \{\} @@ -4806,13 +4806,13 @@ See [HardwareSpec](#v1beta2-hardwarespec). -Workload defines the synthetic traffic model used for every candidate evaluation. +Workload defines the synthetic traffic model used for every candidate evaluation. When omitted, admission supplies a static shape of ISL 1024, OSL 128 and concurrency 10; an explicit empty workload receives the same shape from the schema. An explicit workload.static: \{\} opts out of that shape, leaving the workload to spec.aisimulate.parameters.workload. spec.aisimulate.parameters.workload overrides the fields it names. The default evaluates a fixed concurrency, so it creates neither sweep.max_trials nor a KV-load ratio search. See [WorkloadSpec](#v1beta2-workloadspec). **Validation:** Optional: \{\} -Objective defines scalar optimization or a multi-objective Pareto search. When omitted, the schema supplies the default Optimize mode and SLA; admission supplies the conditional scalar metric. +Objective defines scalar optimization or a multi-objective Pareto search. When omitted, admission supplies the object. Its mode defaults to Optimize, and admission supplies the conditional scalar metric. See [ObjectiveSpec](#objectivespec). **Validation:** Optional: \{\} @@ -4888,13 +4888,13 @@ See [HardwareSpec](#v1beta2-hardwarespec). -Workload defines the synthetic traffic model used for every candidate evaluation. +Workload defines the synthetic traffic model used for every candidate evaluation. When omitted, admission supplies a static shape of ISL 1024, OSL 128 and concurrency 10; an explicit empty workload receives the same shape from the schema. An explicit workload.static: \{\} opts out of that shape, leaving the workload to spec.aisimulate.parameters.workload. spec.aisimulate.parameters.workload overrides the fields it names. The default evaluates a fixed concurrency, so it creates neither sweep.max_trials nor a KV-load ratio search. See [WorkloadSpec](#v1beta2-workloadspec). **Validation:** Optional: \{\} -Objective defines scalar optimization or a multi-objective Pareto search. When omitted, the schema supplies the default Optimize mode and SLA; admission supplies the conditional scalar metric. +Objective defines scalar optimization or a multi-objective Pareto search. When omitted, admission supplies the object. Its mode defaults to Optimize, and admission supplies the conditional scalar metric. See [ObjectiveSpec](#objectivespec). **Validation:** Optional: \{\} @@ -5010,13 +5010,13 @@ See [HardwareSpec](#v1beta2-hardwarespec). -Workload defines the synthetic traffic model used for every candidate evaluation. +Workload defines the synthetic traffic model used for every candidate evaluation. When omitted, admission supplies a static shape of ISL 1024, OSL 128 and concurrency 10; an explicit empty workload receives the same shape from the schema. An explicit workload.static: \{\} opts out of that shape, leaving the workload to spec.aisimulate.parameters.workload. spec.aisimulate.parameters.workload overrides the fields it names. The default evaluates a fixed concurrency, so it creates neither sweep.max_trials nor a KV-load ratio search. See [WorkloadSpec](#v1beta2-workloadspec). **Validation:** Optional: \{\} -Objective defines scalar optimization or a multi-objective Pareto search. When omitted, the schema supplies the default Optimize mode and SLA; admission supplies the conditional scalar metric. +Objective defines scalar optimization or a multi-objective Pareto search. When omitted, admission supplies the object. Its mode defaults to Optimize, and admission supplies the conditional scalar metric. See [ObjectiveSpec](#objectivespec). **Validation:** Optional: \{\} @@ -5209,13 +5209,13 @@ See [HardwareSpec](#v1beta2-hardwarespec). -Workload defines the synthetic traffic model used for every candidate evaluation. +Workload defines the synthetic traffic model used for every candidate evaluation. When omitted, admission supplies a static shape of ISL 1024, OSL 128 and concurrency 10; an explicit empty workload receives the same shape from the schema. An explicit workload.static: \{\} opts out of that shape, leaving the workload to spec.aisimulate.parameters.workload. spec.aisimulate.parameters.workload overrides the fields it names. The default evaluates a fixed concurrency, so it creates neither sweep.max_trials nor a KV-load ratio search. See [WorkloadSpec](#v1beta2-workloadspec). **Validation:** Optional: \{\} -Objective defines scalar optimization or a multi-objective Pareto search. When omitted, the schema supplies the default Optimize mode and SLA; admission supplies the conditional scalar metric. +Objective defines scalar optimization or a multi-objective Pareto search. When omitted, admission supplies the object. Its mode defaults to Optimize, and admission supplies the conditional scalar metric. See [ObjectiveSpec](#objectivespec). **Validation:** Optional: \{\} @@ -5296,7 +5296,7 @@ See [ObjectiveMode](#objectivemode). -Metric is the scalar target when mode is Optimize. When omitted in Optimize mode, admission defaults it to ThroughputPerGPU. +Metric is the scalar target when mode is Optimize. When omitted in Optimize mode, admission defaults it to Throughput. See [ObjectiveMetric](#objectivemetric). **Validation:** Enum: [Throughput ThroughputPerGPU ThroughputPerUser TTFT E2ELatency Goodput GoodputPerGPU] Optional: \{\} @@ -5307,8 +5307,8 @@ See [ObjectiveMetric](#objectivemetric). **Validation:** Enum: [Throughput ThroughputPerGPU ThroughputPerUser TTFT E2ELatency Goodput GoodputPerGPU] MaxItems: 7 MinItems: 2 Optional: \{\} - -SLA defines per-request latency bounds used to calculate goodput. + +SLA defines per-request latency bounds used to calculate goodput. When omitted, no bounds apply. spec.aisimulate.parameters.goal.sla overrides the bounds it names. See [ObjectiveSLASpec](#objectiveslaspec). **Validation:** Optional: \{\} @@ -5407,8 +5407,8 @@ WorkloadSpec defines the synthetic traffic model used for every candidate evalua **Kind:** `type` **Appears in:** [DynamoGraphDeploymentRequestNativeSpec](#v1beta2-dynamographdeploymentrequestnativespec), [DynamoGraphDeploymentRequestSpec](#v1beta2-dynamographdeploymentrequestspec), [DynamoGraphDeploymentRunSpec](#dynamographdeploymentrunspec), [NativeSearchInput](#nativesearchinput) - -Static defines one of AISimulate's synthetic workload shapes. When omitted, it searches the full KV-load ratio range at ISL 4000 and OSL 1000. An explicit empty object opts out of this underlay. + +Static defines one of AISimulate's synthetic workload shapes. Its fields are not defaulted individually: the default belongs to spec.workload as a whole, so an explicit workload object keeps exactly the fields it names. See [StaticWorkloadSpec](#staticworkloadspec). **Validation:** Optional: \{\} diff --git a/pyproject.toml b/pyproject.toml index 6e87d84c8783..86a2962e7d31 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -131,6 +131,7 @@ dynamo = "dynamo.replay.simulation:DynamoReplayRunnerFactory" [project.entry-points."aisimulate.output_adapters"] dgd = "dynamo.aisimulate.output.dgd:create_adapter" +dgdr_run = "dynamo.aisimulate.output.dgdr_run:create_adapter" [dependency-groups] docs = [